<commit_before># You should not use special characters in the value keys
#
# A single whitespace in `line` will match any number of whitespaces. You should be explicit when
# Matching white space in `values`
#
prefixes:
  # jlaunchd doesn't have proper tags. Override the match to use jlaunchd as the tag instead.
  - time_format: "%b %d %H:%M:%S"
    values:
      date: (\w+ +\d+)
      time: (\d\d:\d\d:\d\d)
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      tag: (jlaunchd)
    line: '{date} {time} {hostPrefix}{host} {tag}: '
  - time_format: '%b %d %H:%M:%S'
    values:
      date: (\w+\s+\d+)
      time: (\d\d:\d\d:\d\d)
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      processName: /?(\w+)
      # Most log lines have a process ID, however some do not
      processId: \[?(\d+)?\]?
      tag: ([\w\s]+)
    line: '{date} {time} {hostPrefix}{host} {processName}{processId}: {tag}:'
  - time_format: "%b %d %H:%M:%S"
    values:
      date: (\w+\s+\d+)
      time: (\d\d:\d\d:\d\d)
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      tag: (\w+)
      # Most log lines have a process ID, however some do not
      processId: (\d+)
    line: '{date} {time} {hostPrefix}{host} {tag}[{processId}]:'
  - time_format: "%b %d %H:%M:%S"
    values:
      date: (\w+\s+\d+)
      time: (\d\d:\d\d:\d\d)
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      fpcId: (\d+)
      tag: (\w+)
      # Some logs have data which can be inside brackets or parenthesis
      additionalData: (?:(?:\[|\()(.+)(?:\]|\)))?
    line: '{date} {time} {hostPrefix}{host} fpc{fpcId} {tag}{additionalData}:'
  # The following matches dcpfe specific logs
  - time_format: "%b %d %H:%M:%S"
    values:
      date: (\w+\s+\d+)
      time: (\d\d:\d\d:\d\d)
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      fpcId: (\d+)
      fpcId2: (\d+)
      tag: (\w+)
      # Some logs have data which can be inside brackets or parenthesis
      additionalData: (?:(?:\[|\()(.+)(?:\]|\)))?
    line: '{date} {time} {hostPrefix}{host} fpc{fpcId} fpc{fpcId2} dcpfe: {tag}{additionalData}:'

  # Same structures as above, but to match the ISO8601 date-time format.
    # jlaunchd doesn't have proper tags. Override the match to use jlaunchd as the tag instead.
  - time_format: "%b %d %H:%M:%S"
    values:
      date: (\d{4}-\d{2}-\d{2})
      time: (\d{2}:\d{2}:\d{2}[\.\d{3}]?[\+|-]\d{2}:\d{2})
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      tag: (jlaunchd)
    line: '{date}T{time} {hostPrefix}{host} {tag}: '
  - time_format: '%b %d %H:%M:%S'
    values:
      date: (\d{4}-\d{2}-\d{2})
      time: (\d{2}:\d{2}:\d{2}[\.\d{3}]?[\+|-]\d{2}:\d{2})
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      processName: /?(\w+)
      # Most log lines have a process ID, however some do not
      processId: \[?(\d+)?\]?
      tag: ([\w\s]+)
    line: '{date}T{time} {hostPrefix}{host} {processName}{processId}: {tag}:'
  - time_format: "%b %d %H:%M:%S"
    values:
      date: (\d{4}-\d{2}-\d{2})
      time: (\d{2}:\d{2}:\d{2}[\.\d{3}]?[\+|-]\d{2}:\d{2})
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      tag: (\w+)
      # Most log lines have a process ID, however some do not
      processId: (\d+)
    line: '{date}T{time} {hostPrefix}{host} {tag}[{processId}]:'
  - time_format: "%b %d %H:%M:%S"
    values:
      date: (\d{4}-\d{2}-\d{2})
      time: (\d{2}:\d{2}:\d{2}[\.\d{3}]?[\+|-]\d{2}:\d{2})
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      fpcId: (\d+)
      tag: (\w+)
      # Some logs have data which can be inside brackets or parenthesis
      additionalData: (?:(?:\[|\()(.+)(?:\]|\)))?
    line: '{date}T{time} {hostPrefix}{host} fpc{fpcId} {tag}{additionalData}:'
  # The following matches dcpfe specific logs
  - time_format: "%b %d %H:%M:%S"
    values:
      date: (\d{4}-\d{2}-\d{2})
      time: (\d{2}:\d{2}:\d{2}[\.\d{3}]?[\+|-]\d{2}:\d{2})
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      fpcId: (\d+)
      fpcId2: (\d+)
      tag: (\w+)
      # Some logs have data which can be inside brackets or parenthesis
      additionalData: (?:(?:\[|\()(.+)(?:\]|\)))?
    line: '{date}T{time} {hostPrefix}{host} fpc{fpcId} fpc{fpcId2} dcpfe: {tag}{additionalData}:'<commit_msg>Match another class of Junos FPC messages<commit_after># You should not use special characters in the value keys
#
# A single whitespace in `line` will match any number of whitespaces. You should be explicit when
# Matching white space in `values`
#
prefixes:
  # jlaunchd doesn't have proper tags. Override the match to use jlaunchd as the tag instead.
  - time_format: "%b %d %H:%M:%S"
    values:
      date: (\w+ +\d+)
      time: (\d\d:\d\d:\d\d)
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      tag: (jlaunchd)
    line: '{date} {time} {hostPrefix}{host} {tag}: '
  - time_format: '%b %d %H:%M:%S'
    values:
      date: (\w+\s+\d+)
      time: (\d\d:\d\d:\d\d)
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      processName: /?(\w+)
      # Most log lines have a process ID, however some do not
      processId: \[?(\d+)?\]?
      tag: ([\w\s]+)
    line: '{date} {time} {hostPrefix}{host} {processName}{processId}: {tag}:'
  - time_format: "%b %d %H:%M:%S"
    values:
      date: (\w+\s+\d+)
      time: (\d\d:\d\d:\d\d)
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      tag: (\w+)
      # Most log lines have a process ID, however some do not
      processId: (\d+)
    line: '{date} {time} {hostPrefix}{host} {tag}[{processId}]:'
  - time_format: "%b %d %H:%M:%S"
    values:
      date: (\w+\s+\d+)
      time: (\d\d:\d\d:\d\d)
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      fpcId: (\d+)
      tag: (\w+)
      # Some logs have data which can be inside brackets or parenthesis
      additionalData: (?:(?:\[|\()(.+)(?:\]|\)))?
    line: '{date} {time} {hostPrefix}{host} fpc{fpcId} {tag}{additionalData}:'
  # The following matches dcpfe specific logs
  - time_format: "%b %d %H:%M:%S"
    values:
      date: (\w+\s+\d+)
      time: (\d\d:\d\d:\d\d)
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      fpcId: (\d+)
      fpcId2: (\d+)
      tag: (\w+)
      # Some logs have data which can be inside brackets or parenthesis
      additionalData: (?:(?:\[|\()(.+)(?:\]|\)))?
    line: '{date} {time} {hostPrefix}{host} fpc{fpcId} fpc{fpcId2} dcpfe: {tag}{additionalData}:'
  - time_format: "%b %d %H:%M:%S"
    values:
      date: (\w+\s+\d+)
      time: (\d\d:\d\d:\d\d)
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      tag: (fpc\d)
    line: '{date} {time} {hostPrefix}{host} {tag}'

  # Same structures as above, but to match the ISO8601 date-time format.
    # jlaunchd doesn't have proper tags. Override the match to use jlaunchd as the tag instead.
  - time_format: "%b %d %H:%M:%S"
    values:
      date: (\d{4}-\d{2}-\d{2})
      time: (\d{2}:\d{2}:\d{2}[\.\d{3}]?[\+|-]\d{2}:\d{2})
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      tag: (jlaunchd)
    line: '{date}T{time} {hostPrefix}{host} {tag}: '
  - time_format: '%b %d %H:%M:%S'
    values:
      date: (\d{4}-\d{2}-\d{2})
      time: (\d{2}:\d{2}:\d{2}[\.\d{3}]?[\+|-]\d{2}:\d{2})
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      processName: /?(\w+)
      # Most log lines have a process ID, however some do not
      processId: \[?(\d+)?\]?
      tag: ([\w\s]+)
    line: '{date}T{time} {hostPrefix}{host} {processName}{processId}: {tag}:'
  - time_format: "%b %d %H:%M:%S"
    values:
      date: (\d{4}-\d{2}-\d{2})
      time: (\d{2}:\d{2}:\d{2}[\.\d{3}]?[\+|-]\d{2}:\d{2})
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      tag: (\w+)
      # Most log lines have a process ID, however some do not
      processId: (\d+)
    line: '{date}T{time} {hostPrefix}{host} {tag}[{processId}]:'
  - time_format: "%b %d %H:%M:%S"
    values:
      date: (\d{4}-\d{2}-\d{2})
      time: (\d{2}:\d{2}:\d{2}[\.\d{3}]?[\+|-]\d{2}:\d{2})
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      fpcId: (\d+)
      tag: (\w+)
      # Some logs have data which can be inside brackets or parenthesis
      additionalData: (?:(?:\[|\()(.+)(?:\]|\)))?
    line: '{date}T{time} {hostPrefix}{host} fpc{fpcId} {tag}{additionalData}:'
  # The following matches dcpfe specific logs
  - time_format: "%b %d %H:%M:%S"
    values:
      date: (\d{4}-\d{2}-\d{2})
      time: (\d{2}:\d{2}:\d{2}[\.\d{3}]?[\+|-]\d{2}:\d{2})
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      fpcId: (\d+)
      fpcId2: (\d+)
      tag: (\w+)
      # Some logs have data which can be inside brackets or parenthesis
      additionalData: (?:(?:\[|\()(.+)(?:\]|\)))?
    line: '{date}T{time} {hostPrefix}{host} fpc{fpcId} fpc{fpcId2} dcpfe: {tag}{additionalData}:'
  - time_format: "%b %d %H:%M:%S"
    values:
      date: (\d{4}-\d{2}-\d{2})
      time: (\d{2}:\d{2}:\d{2}[\.\d{3}]?[\+|-]\d{2}:\d{2})
      hostPrefix: (re\d.)?
      host: ([^ ]+)
      tag: (fpc\d)
    line: '{date}T{time} {hostPrefix}{host} {tag}'
{'fraction_non_alphanumeric': 0.25558564658090727, 'fraction_numerical': 0.012976754682915821, 'mean_word_length': 2.429953560371517, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4419800', 'n_tokens_mistral': 3730, 'n_tokens_neox': 3574, 'n_words': 838}