<commit_before>const sourceSchema: any = {
  'http': {
    "title": "Http Source",
    "type": "object",
    "properties": {
      "interface": {
        "type": "string",
        "default": "0.0.0.0",
        "required": true
      },
      "port": {
        "type": "integer",
        "default": 8000,
        "minimum": 1024,
        "maximum": 65535,
        "required": true
      },
      "format": {
        "type": "string",
        "enum": [
          "ActivityStreams"
        ],
        "required": true
      },
      "succeeded-response": {
        "type": "string",
        "default": "ok",
        "required": true
      },
      "server": {
        "type": "object",
        "title": "Server",
        "properties": {
          "max-connections": {
            "type": "integer",
            "default": 1024,
            "required": true
          },
          "request-timeout": {
            "type": "string",
            "default": "10 s",
            "description": "possible units: (s)econd, (m)inute, (h)ours",
            "required": true
          },
          "idle-timeout": {
            "type": "string",
            "default": "60 s",
            "description": "possible units: (s)econd, (m)inute, (h)ours"
          },
          "bind-timeout": {
            "type": "string",
            "default": "1 s",
            "description": "possible units: (s)econd, (m)inute, (h)ours"
          },
          "linger-timeout": {
            "type": "string",
            "default": "1 min",
            "description": "possible units: (s)econd, (m)inute, (h)ours"
          }
        },
        "required": true
      }
    }
  },

  'kafka': {
    "title": "Kafka Source",
    "type": "object",
    "properties": {
      "bootstrap-servers": {
        "type": "string",
        "title": "Bootstrap Servers",
        "default": "",
        "required": true
      },
      "group-id": {
        "type": "string",
        "title": "Group Id",
        "default": ""
      },
      "topics": {
        "type": "array",
        "title": "Subscribed Topics",
        "uniqueItems": true,
        "items": {
          "type": "string",
          "title": "topic"
        },
        "required": true
      },
      "topic-pattern": {
        "type": "string",
        "title": "Subscribed Topic Pattern",
        "description": "supports regex pattern, if \"Subscribed Topics\" is set, will use it instead",
        "default": "",
        "required": true
      },
      "max-concurrent-partitions": {
        "type": "integer",
        "default": 100,
        "required": true
      },
      "commit-max-batches": {
        "type": "integer",
        "default": 20,
        "required": true
      },
      "use-dispatcher": {
        "type": "string",
        "default": "akka.kafka.default-dispatcher"
      },
      "poll-interval": {
        "type": "string",
        "default": "50ms",
        "required": true
      },
      "poll-timeout": {
        "type": "string",
        "default": "50ms"
      },
      "stop-timeout": {
        "type": "string",
        "default": "30s"
      },
      "close-timeout": {
        "type": "string",
        "default": "20s"
      },
      "commit-timeout": {
        "type": "string",
        "default": "15s"
      },
      "commit-time-warning": {
        "type": "string",
        "default": "1s"
      },
      "wakeup-timeout": {
        "type": "string",
        "default": "3s",
        "required": true
      },
      "max-wakeups": {
        "type": "integer",
        "default": 10,
        "required": true
      },
      "properties": {
        "type": "object",
        "title": "Kafka Consumer Properties",
        "properties": {
          /*"enable.auto.commit": {
            "type": "boolean",
            "title": "Enable auto commit?",
            "default": false
          }*/
        },
        "required": true
      }
    }
  }
}

const transformSchema: any = {
  'tnc-topic-resolver': {
    "title": "TNC Topic Resolver Transform",
    "type": "object",
    "properties": {
      "default-topic": {
        "type": "string",
        "title": "Default Topic",
        "description": "supports substitutes: {app_name}, {app_env}, {story_name}",
        "default": "event-{app_name}-default",
        "required": true
      }
    }
  }
}<commit_msg>add tnc-event-filter support in backend<commit_after>const sourceSchema: any = {
  'http': {
    "title": "Http Source",
    "type": "object",
    "properties": {
      "interface": {
        "type": "string",
        "default": "0.0.0.0",
        "required": true
      },
      "port": {
        "type": "integer",
        "default": 8000,
        "minimum": 1024,
        "maximum": 65535,
        "required": true
      },
      "format": {
        "type": "string",
        "enum": [
          "ActivityStreams"
        ],
        "required": true
      },
      "succeeded-response": {
        "type": "string",
        "default": "ok",
        "required": true
      },
      "server": {
        "type": "object",
        "title": "Server",
        "properties": {
          "max-connections": {
            "type": "integer",
            "default": 1024,
            "required": true
          },
          "request-timeout": {
            "type": "string",
            "default": "10 s",
            "description": "possible units: (s)econd, (m)inute, (h)ours",
            "required": true
          },
          "idle-timeout": {
            "type": "string",
            "default": "60 s",
            "description": "possible units: (s)econd, (m)inute, (h)ours"
          },
          "bind-timeout": {
            "type": "string",
            "default": "1 s",
            "description": "possible units: (s)econd, (m)inute, (h)ours"
          },
          "linger-timeout": {
            "type": "string",
            "default": "1 min",
            "description": "possible units: (s)econd, (m)inute, (h)ours"
          }
        },
        "required": true
      }
    }
  },

  'kafka': {
    "title": "Kafka Source",
    "type": "object",
    "properties": {
      "bootstrap-servers": {
        "type": "string",
        "title": "Bootstrap Servers",
        "default": "",
        "required": true
      },
      "group-id": {
        "type": "string",
        "title": "Group Id",
        "default": ""
      },
      "topics": {
        "type": "array",
        "title": "Subscribed Topics",
        "uniqueItems": true,
        "items": {
          "type": "string",
          "title": "topic"
        },
        "required": true
      },
      "topic-pattern": {
        "type": "string",
        "title": "Subscribed Topic Pattern",
        "description": "supports regex pattern, if \"Subscribed Topics\" is set, will use it instead",
        "default": "",
        "required": true
      },
      "max-concurrent-partitions": {
        "type": "integer",
        "default": 100,
        "required": true
      },
      "commit-max-batches": {
        "type": "integer",
        "default": 20,
        "required": true
      },
      "use-dispatcher": {
        "type": "string",
        "default": "akka.kafka.default-dispatcher"
      },
      "poll-interval": {
        "type": "string",
        "default": "50ms",
        "required": true
      },
      "poll-timeout": {
        "type": "string",
        "default": "50ms"
      },
      "stop-timeout": {
        "type": "string",
        "default": "30s"
      },
      "close-timeout": {
        "type": "string",
        "default": "20s"
      },
      "commit-timeout": {
        "type": "string",
        "default": "15s"
      },
      "commit-time-warning": {
        "type": "string",
        "default": "1s"
      },
      "wakeup-timeout": {
        "type": "string",
        "default": "3s",
        "required": true
      },
      "max-wakeups": {
        "type": "integer",
        "default": 10,
        "required": true
      },
      "properties": {
        "type": "object",
        "title": "Kafka Consumer Properties",
        "properties": {
          /*"enable.auto.commit": {
            "type": "boolean",
            "title": "Enable auto commit?",
            "default": false
          }*/
        },
        "required": true
      }
    }
  }
}

const transformSchema: any = {
  'tnc-topic-resolver': {
    "title": "TNC Topic Resolver Transform",
    "type": "object",
    "properties": {
      "default-topic": {
        "type": "string",
        "title": "Default Topic",
        "description": "supports substitutes: {app_name}, {app_env}, {story_name}",
        "default": "event-{app_name}-default",
        "required": true
      }
    }
  },
  'tnc-event-filter': {
    "title": "TNC Event Filter",
    "type": "object",
    "properties": {
      "event-name-white-list": {
        "type": "array",
        "title": "Event Name White List",
        "uniqueItems": true,
        "items": {
          "type": "string",
          "title": "Event Name Regex"
        },
        "required": true
      },
      "event-name-black-list": {
        "type": "array",
        "title": "Event Name Black List",
        "uniqueItems": true,
        "items": {
          "type": "string",
          "title": "Event Name Regex"
        },
        "required": true
      },
      "channel-white-list": {
        "type": "array",
        "title": "Channel White List",
        "uniqueItems": true,
        "items": {
          "type": "string",
          "title": "Channel Regex"
        },
        "required": true
      },
      "channel-black-list": {
        "type": "array",
        "title": "Channel Black List",
        "uniqueItems": true,
        "items": {
          "type": "string",
          "title": "Channel Regex"
        },
        "required": true
      },
      "allowed-transport-modes": {
        "type": "array",
        "title": "Allowed Transport Modes",
        "uniqueItems": true,
        "items": {
          "type": "string",
          "enum": [
            "SYNC_PLUS", "ASYNC", "BOTH", "NONE"
          ],
          "title": "Transport Mode"
        },
        "default": ["ASYNC", "BOTH", "NONE"],
        "required": true
      },
      "only-extras": {
        "type": "object",
        "title": "Only Matched Extra Infos",
        "required": true,
        "properties": {}
      }
    }
  }
}
{'fraction_non_alphanumeric': 0.1855103222072159, 'fraction_numerical': 0.008875168821146055, 'mean_word_length': 1.5359589041095891, 'pattern_counts': {'":': 308, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2961155', 'n_tokens_mistral': 3229, 'n_tokens_neox': 3149, 'n_words': 681}