<commit_before>require=(function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);throw new Error("Cannot find module '"+o+"'")}var f=n[o]={exports:{}};t[o][0].call(f.exports,function(e){var n=t[o][1][e];return s(n?n:e)},f,f.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o<r.length;o++)s(r[o]);return s})({"bRhrlU":[function(require,module,exports){
var validator = require('jsonschema').Validator;
var v = new validator();

var headerSchema = {
  "id" : "Header",
  "type": "object",
  "properties": {
    "to": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "uniqueItems": true,
      "minItems": 1
    },
    "category": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "uniqueItems": true,
      "minItems": 1
    },
    "section": {
      "type": "object",
      "patternProperties": {
        ".+": {
          "type": "string"
        }
      }
    },
    "sub": {
      "type": "object",
      "patternProperties": {
        ".+": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "uniqueItems": true,
          "minItems": 1
        }
      }
    },
    "unique_args": {
      "type": "object",
      "patternProperties": {
        ".+": {
          "type": "string"
        }
      }
    },
    "filters": {
      "type": "object",
      "patternProperties": {
        ".+": {
          "type": "object",
          "properties": {
            "settings": {
              "required": true,
              "type": "object",
              "patternProperties": {
                ".+": {
                  "type": ["string", "number"]
                }
              }
            }
          }
        }
      }<commit_msg>Allow duplicates in the Sub header<commit_after>require=(function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);throw new Error("Cannot find module '"+o+"'")}var f=n[o]={exports:{}};t[o][0].call(f.exports,function(e){var n=t[o][1][e];return s(n?n:e)},f,f.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o<r.length;o++)s(r[o]);return s})({"bRhrlU":[function(require,module,exports){
var validator = require('jsonschema').Validator;
var v = new validator();

var headerSchema = {
  "id" : "Header",
  "type": "object",
  "properties": {
    "to": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "uniqueItems": true,
      "minItems": 1
    },
    "category": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "uniqueItems": true,
      "minItems": 1
    },
    "section": {
      "type": "object",
      "patternProperties": {
        ".+": {
          "type": "string"
        }
      }
    },
    "sub": {
      "type": "object",
      "patternProperties": {
        ".+": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "minItems": 1
        }
      }
    },
    "unique_args": {
      "type": "object",
      "patternProperties": {
        ".+": {
          "type": "string"
        }
      }
    },
    "filters": {
      "type": "object",
      "patternProperties": {
        ".+": {
          "type": "object",
          "properties": {
            "settings": {
              "required": true,
              "type": "object",
              "patternProperties": {
                ".+": {
                  "type": ["string", "number"]
                }
              }
            }
          }
        }
      }
{'fraction_non_alphanumeric': 0.21764705882352942, 'fraction_numerical': 0.0042780748663101605, 'mean_word_length': 1.838391502276176, 'pattern_counts': {'":': 91, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6868656', 'n_tokens_mistral': 1278, 'n_tokens_neox': 1262, 'n_words': 193}