<commit_before>spec:
  x-sub-request-filters:
    - type: default
      name: http
      options:
        allow:
          - pattern: /^https?:\/\//
            forward_headers:
              user-agent: true
  title: The Change Propagation root
  paths:
    /{api:sys}/links:
      x-modules:
        - path: sys/dep_updates.js
          options:
            templates:
              mw_api:
                uri: 'http://localhost<%= scope['::port_fragment'] %>/w/api.php'
                headers:
                  host: '{{message.meta.domain}}'
                body:
                  formatversion: 2
    /{api:sys}/queue:
      x-modules:
        - path: sys/kafka.js
          options:
            metadata_broker_list: localhost:9092
            dc_name: datacenter1
            concurrency: 5  # keep the concurrency modest
            startup_delay: 10000
            templates:
              summary_rerender:
                topic: resource_change
                match:
                  meta:
                    uri: '/^https?:\/\/[^\/]+\/api\/rest_v1\/page\/html\/([^\/]+)$/'
                  tags:
                    - restbase
                exec:
                  method: get
                  # Don't encode title since it should be already encoded
                  uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/summary/{{match.meta.uri[1]}}'
                  query:
                    redirect: false
                  headers:
                    cache-control: no-cache

              mobile_rerender:
                topic: resource_change
                match:
                  meta:
                    uri: '/^https?:\/\/[^\/]+\/api\/rest_v1\/page\/html\/([^\/]+)$/'
                  tags:
                    - restbase
                exec:
                  method: get
                  # Don't encode title since it should be already encoded
                  uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/mobile-sections/{{match.meta.uri[1]}}'
                  query:
                    redirect: false
                  headers:
                    cache-control: no-cache

              # RESTBase update jobs
              mw_purge:
                topic: resource_change
                match:
                  meta:
                    uri: '/^https?:\/\/[^\/]+\/wiki\/(?<title>.+)$/'
                  tags:
                    - purge
                exec:
                  method: get
                  uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/html/{{match.meta.uri.title}}'
                  headers:
                    cache-control: no-cache
                    if-unmodified-since: '{{date(message.meta.dt)}}'
                  query:
                    redirect: false

              null_edit:
                topic: resource_change
                ignore:
                  status:
                    - 403
                    - 412
                match:
                  meta:
                    uri: '/^https?:\/\/[^\/]+\/wiki\/(?<title>.+)$/'
                  tags:
                    - null_edit
                exec:
                  method: get
                  uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/html/{{match.meta.uri.title}}'
                  headers:
                    cache-control: no-cache
                    if-unmodified-since: '{{date(message.meta.dt)}}'
                  query:
                    redirect: false

              page_edit:
                topic: mediawiki.revision_create
                retry_on:
                  status:
                    - '5xx'
                    - 404
                exec:
                  method: get
                  uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/html/{message.page_title}/{{message.rev_id}}'
                  headers:
                    cache-control: no-cache
                    x-restbase-parentrevision: '{{message.rev_parent_id}}'
                    if-unmodified-since: '{{date(message.meta.dt)}}'
                  query:
                    redirect: false

              revision_visibility_change:
                topic: mediawiki.revision_visibility_set
                ignore:
                  status:
                    - 403
                    - 412
                exec:
                  method: get
                  uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/revision/{{message.revision_id}}'
                  headers:
                    cache-control: no-cache
                  query:
                    redirect: false

              page_delete:
                topic: mediawiki.page_delete
                ignore:
                  status:
                    - 404 # 404 is a normal response for page deletion
                    - 412
                exec:
                  method: get
                  uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/title/{message.title}'
                  headers:
                    cache-control: no-cache
                  query:
                    redirect: false

              page_restore:
                topic: mediawiki.page_restore
                exec:
                  method: get
                  uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/html/{message.title}'
                  headers:
                    cache-control: no-cache
                  query:
                    redirect: false

              page_move:
                topic: mediawiki.page_move
                exec:
                  - method: get
                    uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/html/{message.new_title}/{{message.new_revision_id}}'
                    headers:
                      cache-control: no-cache
                      if-unmodified-since: '{{date(message.meta.dt)}}'
                    query:
                      redirect: false
                  - method: get
                    uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/title/{message.old_title}'
                    headers:
                      cache-control: no-cache
                    query:
                      redirect: false

              transclusion_update:
                topic: mediawiki.revision_create
                exec:
                  method: 'post'
                  uri: '/sys/links/transcludes/{message.page_title}'
                  body: '{{globals.message}}'

              on_transclusion_update:
                concurrency: 10
                topic: resource_change
                match:
                  meta:
                    uri: '/https?:\/\/[^\/]+\/wiki\/(?<title>.+)/'
                  tags: [ 'change-prop', 'transcludes' ]
                exec:
                  method: get
                  uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/html/{{match.meta.uri.title}}'
                  headers:
                    cache-control: no-cache
                    if-unmodified-since: '{{date(message.meta.dt)}}'
                  query:
                    redirect: false<commit_msg>Change-Prop: Switch to new events.<commit_after>spec:
  x-sub-request-filters:
    - type: default
      name: http
      options:
        allow:
          - pattern: /^https?:\/\//
            forward_headers:
              user-agent: true
  title: The Change Propagation root
  paths:
    /{api:sys}/links:
      x-modules:
        - path: sys/dep_updates.js
          options:
            templates:
              mw_api:
                uri: 'http://localhost<%= scope['::port_fragment'] %>/w/api.php'
                headers:
                  host: '{{message.meta.domain}}'
                body:
                  formatversion: 2
    /{api:sys}/queue:
      x-modules:
        - path: sys/kafka.js
          options:
            metadata_broker_list: localhost:9092
            dc_name: datacenter1
            concurrency: 5  # keep the concurrency modest
            startup_delay: 10000
            templates:
              summary_rerender:
                topic: resource_change
                match:
                  meta:
                    uri: '/^https?:\/\/[^\/]+\/api\/rest_v1\/page\/html\/([^\/]+)$/'
                  tags:
                    - restbase
                exec:
                  method: get
                  # Don't encode title since it should be already encoded
                  uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/summary/{{match.meta.uri[1]}}'
                  query:
                    redirect: false
                  headers:
                    cache-control: no-cache

              mobile_rerender:
                topic: resource_change
                match:
                  meta:
                    uri: '/^https?:\/\/[^\/]+\/api\/rest_v1\/page\/html\/([^\/]+)$/'
                  tags:
                    - restbase
                exec:
                  method: get
                  # Don't encode title since it should be already encoded
                  uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/mobile-sections/{{match.meta.uri[1]}}'
                  query:
                    redirect: false
                  headers:
                    cache-control: no-cache

              # RESTBase update jobs
              mw_purge:
                topic: resource_change
                match:
                  meta:
                    uri: '/^https?:\/\/[^\/]+\/wiki\/(?<title>.+)$/'
                  tags:
                    - purge
                exec:
                  method: get
                  uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/html/{{match.meta.uri.title}}'
                  headers:
                    cache-control: no-cache
                    if-unmodified-since: '{{date(message.meta.dt)}}'
                  query:
                    redirect: false

              null_edit:
                topic: resource_change
                ignore:
                  status:
                    - 403
                    - 412
                match:
                  meta:
                    uri: '/^https?:\/\/[^\/]+\/wiki\/(?<title>.+)$/'
                  tags:
                    - null_edit
                exec:
                  method: get
                  uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/html/{{match.meta.uri.title}}'
                  headers:
                    cache-control: no-cache
                    if-unmodified-since: '{{date(message.meta.dt)}}'
                  query:
                    redirect: false

              page_edit:
                topic: mediawiki.revision-create
                retry_on:
                  status:
                    - '5xx'
                    - 404
                exec:
                  method: get
                  uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/html/{message.page_title}/{{message.rev_id}}'
                  headers:
                    cache-control: no-cache
                    x-restbase-parentrevision: '{{message.rev_parent_id}}'
                    if-unmodified-since: '{{date(message.meta.dt)}}'
                  query:
                    redirect: false

              revision_visibility_change:
                topic: mediawiki.evision-visibility-change
                ignore:
                  status:
                    - 403
                    - 412
                exec:
                  method: get
                  uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/revision/{{message.rev_id}}'
                  headers:
                    cache-control: no-cache
                  query:
                    redirect: false

              page_delete:
                topic: mediawiki.page-delete
                ignore:
                  status:
                    - 404 # 404 is a normal response for page deletion
                    - 412
                exec:
                  method: get
                  uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/title/{message.page_title}'
                  headers:
                    cache-control: no-cache
                  query:
                    redirect: false

              page_restore:
                topic: mediawiki.page-undelete
                exec:
                  method: get
                  uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/html/{message.page_title}'
                  headers:
                    cache-control: no-cache
                  query:
                    redirect: false

              page_move:
                topic: mediawiki.page-move
                exec:
                  - method: get
                    uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/html/{message.page_title}/{{message.rev_id}}'
                    headers:
                      cache-control: no-cache
                      if-unmodified-since: '{{date(message.meta.dt)}}'
                    query:
                      redirect: false
                  - method: get
                    uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/title/{message.prior_state.page_title}'
                    headers:
                      cache-control: no-cache
                    query:
                      redirect: false

              transclusion_update:
                topic: mediawiki.revision-create
                exec:
                  method: 'post'
                  uri: '/sys/links/transcludes/{message.page_title}'
                  body: '{{globals.message}}'

              on_transclusion_update:
                concurrency: 10
                topic: change-prop.transcludes.continue
                match:
                  meta:
                    uri: '/https?:\/\/[^\/]+\/wiki\/(?<title>.+)/'
                  tags: [ 'change-prop', 'transcludes' ]
                exec:
                  method: get
                  uri: '<%= @restbase_uri %>/{{message.meta.domain}}/v1/page/html/{{match.meta.uri.title}}'
                  headers:
                    cache-control: no-cache
                    if-unmodified-since: '{{date(message.meta.dt)}}'
                  query:
                    redirect: false
{'fraction_non_alphanumeric': 0.11491356987892784, 'fraction_numerical': 0.007558261599832038, 'mean_word_length': 0.9410486280901929, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3102413', 'n_tokens_mistral': 3849, 'n_tokens_neox': 3371, 'n_words': 633}