<commit_before>config-file: vast.conf

fixtures:
  BiTopologyTester:
    enter: | # python
      node0 = Server(self.cmd,
                     ['-e', f':{VAST_PORT}', '-i', 'node0', 'start'],
                     work_dir, name='node0', port=VAST_PORT,
                     config_file=f'{self.config_file}')
      node1 = Server(self.cmd,
                     ['-e', ':42124',
                      '-i', 'node1', 'start'],
                     work_dir, name='node1', port=42124,
                     config_file=f'{self.config_file}')
      cmd += ['-e', f':{VAST_PORT}']

    exit: | # python
      node0.stop()
      node1.stop()

  ServerTester:
    enter: | # python
      node = Server(self.cmd,
                    ['-e', f':{VAST_PORT}', '-i', 'node', 'start'],
                    work_dir, name='node', port=VAST_PORT,
                    config_file=f'{self.config_file}')
      cmd += ['-e', f':{VAST_PORT}']


    exit: | # python
      node.stop()


tests:
  JSON schema inference:
    tags: [schema, infer, json]
    steps:
      - command: infer
        input: data/json/empty-object.json
      - command: infer
        input: data/json/basic-types.json
      - command: infer
        input: data/json/string-types.json
      - command: infer
        input: data/json/array-types.json
      - command: infer
        input: data/json/nested-object.json
  Conn log counting:
    tags: [node, counting, zeek]
    steps:
      - command: -N import -b zeek
        input: data/zeek/conn.log.gz
      - command: -N count ":addr == 192.168.1.104"
      - command: -N count -s ":addr == 192.168.1.104"
  Node Zeek conn log:
    tags: [node, import-export, zeek]
    steps:
      - command: -N import zeek
        input: data/zeek/conn.log.gz
      - command: -N export ascii 'resp_h == 192.168.1.104'
      - command: -N export ascii 'orig_bytes > 1k && orig_bytes < 1Ki'
      - command: -N export ascii ':string == "OrfTtuI5G4e" || :port == 67/udp'
      - command: -N export ascii '#type == "zeek.conn" && resp_h == 192.168.1.104'
      - command: -N export ascii '#type != "zeek.conn" && \#type != "vast.statistics"'
      - command: -N export ascii '#type != "foobar" && resp_h == 192.168.1.104'
  Node Zeek multiple imports:
    tags: [node, import-export, zeek]
    steps:
      - command: -N import zeek
        input: data/zeek/conn.log.gz
      - command: -N import zeek
        input: data/zeek/dns.log.gz
      - command: -N export ascii 'resp_h == 192.168.1.104'
      - command: -N export ascii 'zeek.conn.id.resp_h == 192.168.1.104'
  Node Zeek dns log:
    tags: [node, import-export, zeek]
    steps:
      - command: -N import zeek
        input: data/zeek/dns.log.gz
      - command: -N export ascii 'resp_h == 192.168.1.104'
      - command: -N export zeek 'resp_h == 192.168.1.104'
        transformation: awk '!/^#(open|close)/'
  Node Zeek http log:
    tags: [node, import-export, zeek]
    steps:
      - command: -N import zeek
        input: data/zeek/http.log.gz
      - command: -N export ascii 'resp_h == 172.16.31.10'
  Node Zeek snmp log:
    tags: [node, import-export, zeek]
    steps:
      - command: -N import zeek
        input: data/zeek/snmp.log.gz
      - command: -N export ascii 'duration >= 3s'
  # Ground truth established via
  #     zcat < input | bgpdump -m -  \
  #       | awk -F '|' '$6 ~ /^103\.11\./ { print $6 }'
  # ...plus some piping to sort and uniq.
  Node Basic mrt:
    tags: [node, import-export, mrt, disabled]
    steps:
      - command: -N import mrt
        input: data/mrt/updates.20180530.0000.gz
      # This prefix occurs 93 times.
      - command: -N export ascii 'prefix == 192.168.3.11/23'
      # There are 53 entries for 172.16.31.10/24. All of them should include the
      # IP address 172.16.58.3.
      - command: -N export ascii '172.16.58.3 in prefix'
      # Historical component for query in issue #201. No results.
      - command: -N export ascii '172.16.58.3 in prefix'
      - command: -N export ascii 'mrt.bgp4mp.update.announcement.prefix == 192.168.3.11/23'
  Export json:
    tags: [json, zeek]
    steps:
      - command: -N import zeek
        input: data/zeek/conn.log.gz
      - command: -N export json 'resp_h == 192.168.1.104'

  Server Zeek conn log:
    tags: [server, import-export, zeek]
    fixture: ServerTester
    steps:
      - command: import -b zeek
        input: data/zeek/conn.log.gz
      - command: export ascii 'resp_h == 192.168.1.104'
      # import some more to make sure accounting data is in the system.
      - command: import -b zeek
        input: data/zeek/conn.log.gz
      - command: import -b zeek
        input: data/zeek/conn.log.gz
      - command: import -b zeek
        input: data/zeek/conn.log.gz
      - command: import -b zeek
        input: data/zeek/conn.log.gz
      - command: status
        transformation: jq '.node.index.statistics.layouts | del(."vast.statistics".count)'
  Server Basic bgpdump:
    tags: [server, import-export, bgpdump, disabled]
    fixture: ServerTester
    steps:
      - command: import -b bgpdump
        input: data/bgpdump/updates.20180530.bgpdump.gz
      - command: export ascii '172.16.58.3 in prefix'
      - command: export ascii '192.168.3.11 in prefix'
  Query Operators:
    tags: [server, operator]
    fixture: ServerTester
    steps:
      - command: import -b zeek
        input: data/zeek/conn.log.gz
      - command: export ascii 'conn.duration <= 1.0s'
      - command: export ascii 'duration >= 10.0s'
      - command: export ascii 'duration < 2.0s'
      - command: export ascii 'service  == "smtp"'
      - command: export ascii 'missed_bytes  != 0'
      - command: export ascii 'id.orig_h !in 192.168.1.0/24'
      - command: export ascii 'id.orig_h in fe80:5074:1b53:7e7::/64'
      - command: export ascii 'id.orig_h ni fe80:5074:1b53:7e7::/64'
  Expressions:
    tags: [server, expression]
    fixture: ServerTester
    steps:
      - command: import -b zeek
        input: data/zeek/conn.log.gz
      - command: export ascii 'fe80::5074:1b53:7e7:ad4d || 169.254.225.22'
      - command: export ascii '"OrfTtuI5G4e" || fe80::5074:1b53:7e7:ad4d'
  Node Type Query:
    tags: [node, type, ch6104]
    steps:
      - command: -N import -n 20 zeek
        input: data/zeek/conn.log.gz
      - command: -N export ascii '#type == "zeek.conn"'
  Type Query:
    tags: [server, type, ch5404]
    fixture: ServerTester
    steps:
      - command: import -b -n 20 zeek
        input: data/zeek/conn.log.gz
      - command: export ascii '#type == "zeek.conn"'
  Node json zeek conn:
    tags: [node, import-export, zeek, json]
    steps:
      - command: -N import json -s @./misc/schema/zeek-conn.schema -t zeek.conn
        input: data/json/conn.log.json.gz
      - command: -N export ascii 'duration > 6s'
      - command: -N export ascii '#timestamp >= 2011-08-15T03:48'
  Node suricata alert:
    tags: [node, import-export, suricata, eve]
    steps:
      - command: -N import suricata -r @./data/suricata/eve.json
      - command: -N export ascii 'src_ip == 172.16.17.32'
      - command: -N export csv '#type ~ /suricata.*/'

  Node argus csv:
    tags: [node, import-export, argus, csv]
    steps:
      - command: -N import csv -t argus.record
        input: data/csv/argus-M57-10k-pkts.csv.gz
      - command: -N export ascii 'State != "CON" && Dur > 4900ms'
      - command: -N export ascii 'Cause == "Status" && Dur > 1s'

  Manual 1:
    condition: 'import -h 2>&1 | grep -q pcap'
    tags: [examples]
    fixture: ServerTester
    steps:
      - command: import -b pcap
        input: data/pcap/example.pcap.gz
      - command: export ascii ':addr in 192.168.168.0/24'
      - command: export pcap 'sport > 60000/tcp && src !in 192.168.168.0/8'

  Manual 1 node:
    condition: 'import -h 2>&1 | grep -q pcap'
    tags: [examples]
    steps:
      - command: -N import pcap
        input: data/pcap/example.pcap.gz
      - command: -N export ascii ':addr in 192.168.168.0/24'
      - command: -N export pcap 'sport > 60000/tcp && src !in 192.168.168.0/8'
      - command: -N status
        transformation: jq '.node.index.statistics'

  Manual 2:
    tags: [examples, disabled]
    fixture: BiTopologyTester
    steps:
      - command: peer 'localhost:42124'
      - command: status

  Importing VLAN traffic:
    condition: 'import -h 2>&1 | grep -q pcap'
    tags: [import, pcap]
    steps:
      - command: -N import pcap
        input: data/pcap/vlan.pcap.gz
      - command: -N status
        transformation: jq '.node.index.statistics'

  Multi addr query:
    tags: [import-export, zeek]
    steps:
      - command: -N import zeek
        input: data/zeek/conn.log.gz
      - command: -N export ascii
        input: queries/multi_addr.txt

  Pivot:
    tags: [pivot]
    steps:
      - command: -N import suricata
        input: data/pcap/suricata/eve.json.gz
      - command: -N import zeek
        input: data/pcap/zeek/conn.log.gz
      - command: -N import zeek
        input: data/pcap/zeek/files.log.gz
      - command: -N import zeek
        input: data/pcap/zeek/http.log.gz
      - command: -N import zeek
        input: data/pcap/zeek/packet_filter.log.gz
      - command: -N import zeek
        input: data/pcap/zeek/ssh.log.gz
      - command: -N pivot zeek.conn '#type ~ /suricata.*/ && dest_ip == 192.168.127.12'
        transformation: awk '!/^#(open|close)/'
      - command: -N pivot zeek.conn '#type ~ /zeek.http/ && method == "GET"'
        transformation: awk '!/^#(open|close)/'
      - guard: 'import -h 2>&1 | grep -q pcap'
      - command: -N import pcap
        input: data/pcap/example.pcap.gz
      - command: -N pivot pcap.packet 'dest_ip == 192.168.127.12'
        transformation: tcpdump -n -tt -r -
  Arrow:
    condition: 'export -h 2>&1 | grep -q arrow'
    tags: [export, arrow]
    steps:
      - command: -N import zeek
        input: data/zeek/conn.log.gz
      - command: -N export -n 10 arrow '#type == "zeek.conn"'
        transformation: python @./misc/scripts/print-arrow.py
  Type Registry:
    tags: [import]
    steps:
      - command: -N import -n 1 test
      - command: -N status
        transformation: jq '.node."type-registry".statistics'

<commit_msg>Fix malformatted YAML in integration test<commit_after>config-file: vast.conf

fixtures:
  BiTopologyTester:
    enter: | # python
      node0 = Server(self.cmd,
                     ['-e', f':{VAST_PORT}', '-i', 'node0', 'start'],
                     work_dir, name='node0', port=VAST_PORT,
                     config_file=f'{self.config_file}')
      node1 = Server(self.cmd,
                     ['-e', ':42124',
                      '-i', 'node1', 'start'],
                     work_dir, name='node1', port=42124,
                     config_file=f'{self.config_file}')
      cmd += ['-e', f':{VAST_PORT}']

    exit: | # python
      node0.stop()
      node1.stop()

  ServerTester:
    enter: | # python
      node = Server(self.cmd,
                    ['-e', f':{VAST_PORT}', '-i', 'node', 'start'],
                    work_dir, name='node', port=VAST_PORT,
                    config_file=f'{self.config_file}')
      cmd += ['-e', f':{VAST_PORT}']


    exit: | # python
      node.stop()


tests:
  JSON schema inference:
    tags: [schema, infer, json]
    steps:
      - command: infer
        input: data/json/empty-object.json
      - command: infer
        input: data/json/basic-types.json
      - command: infer
        input: data/json/string-types.json
      - command: infer
        input: data/json/array-types.json
      - command: infer
        input: data/json/nested-object.json
  Conn log counting:
    tags: [node, counting, zeek]
    steps:
      - command: -N import -b zeek
        input: data/zeek/conn.log.gz
      - command: -N count ":addr == 192.168.1.104"
      - command: -N count -s ":addr == 192.168.1.104"
  Node Zeek conn log:
    tags: [node, import-export, zeek]
    steps:
      - command: -N import zeek
        input: data/zeek/conn.log.gz
      - command: -N export ascii 'resp_h == 192.168.1.104'
      - command: -N export ascii 'orig_bytes > 1k && orig_bytes < 1Ki'
      - command: -N export ascii ':string == "OrfTtuI5G4e" || :port == 67/udp'
      - command: -N export ascii '#type == "zeek.conn" && resp_h == 192.168.1.104'
      - command: '-N export ascii "#type != \"zeek.conn\" && #type != \"vast.statistics\""'
      - command: -N export ascii '#type != "foobar" && resp_h == 192.168.1.104'
  Node Zeek multiple imports:
    tags: [node, import-export, zeek]
    steps:
      - command: -N import zeek
        input: data/zeek/conn.log.gz
      - command: -N import zeek
        input: data/zeek/dns.log.gz
      - command: -N export ascii 'resp_h == 192.168.1.104'
      - command: -N export ascii 'zeek.conn.id.resp_h == 192.168.1.104'
  Node Zeek dns log:
    tags: [node, import-export, zeek]
    steps:
      - command: -N import zeek
        input: data/zeek/dns.log.gz
      - command: -N export ascii 'resp_h == 192.168.1.104'
      - command: -N export zeek 'resp_h == 192.168.1.104'
        transformation: awk '!/^#(open|close)/'
  Node Zeek http log:
    tags: [node, import-export, zeek]
    steps:
      - command: -N import zeek
        input: data/zeek/http.log.gz
      - command: -N export ascii 'resp_h == 172.16.31.10'
  Node Zeek snmp log:
    tags: [node, import-export, zeek]
    steps:
      - command: -N import zeek
        input: data/zeek/snmp.log.gz
      - command: -N export ascii 'duration >= 3s'
  # Ground truth established via
  #     zcat < input | bgpdump -m -  \
  #       | awk -F '|' '$6 ~ /^103\.11\./ { print $6 }'
  # ...plus some piping to sort and uniq.
  Node Basic mrt:
    tags: [node, import-export, mrt, disabled]
    steps:
      - command: -N import mrt
        input: data/mrt/updates.20180530.0000.gz
      # This prefix occurs 93 times.
      - command: -N export ascii 'prefix == 192.168.3.11/23'
      # There are 53 entries for 172.16.31.10/24. All of them should include the
      # IP address 172.16.58.3.
      - command: -N export ascii '172.16.58.3 in prefix'
      # Historical component for query in issue #201. No results.
      - command: -N export ascii '172.16.58.3 in prefix'
      - command: -N export ascii 'mrt.bgp4mp.update.announcement.prefix == 192.168.3.11/23'
  Export json:
    tags: [json, zeek]
    steps:
      - command: -N import zeek
        input: data/zeek/conn.log.gz
      - command: -N export json 'resp_h == 192.168.1.104'

  Server Zeek conn log:
    tags: [server, import-export, zeek]
    fixture: ServerTester
    steps:
      - command: import -b zeek
        input: data/zeek/conn.log.gz
      - command: export ascii 'resp_h == 192.168.1.104'
      # import some more to make sure accounting data is in the system.
      - command: import -b zeek
        input: data/zeek/conn.log.gz
      - command: import -b zeek
        input: data/zeek/conn.log.gz
      - command: import -b zeek
        input: data/zeek/conn.log.gz
      - command: import -b zeek
        input: data/zeek/conn.log.gz
      - command: status
        transformation: jq '.node.index.statistics.layouts | del(."vast.statistics".count)'
  Server Basic bgpdump:
    tags: [server, import-export, bgpdump, disabled]
    fixture: ServerTester
    steps:
      - command: import -b bgpdump
        input: data/bgpdump/updates.20180530.bgpdump.gz
      - command: export ascii '172.16.58.3 in prefix'
      - command: export ascii '192.168.3.11 in prefix'
  Query Operators:
    tags: [server, operator]
    fixture: ServerTester
    steps:
      - command: import -b zeek
        input: data/zeek/conn.log.gz
      - command: export ascii 'conn.duration <= 1.0s'
      - command: export ascii 'duration >= 10.0s'
      - command: export ascii 'duration < 2.0s'
      - command: export ascii 'service  == "smtp"'
      - command: export ascii 'missed_bytes  != 0'
      - command: export ascii 'id.orig_h !in 192.168.1.0/24'
      - command: export ascii 'id.orig_h in fe80:5074:1b53:7e7::/64'
      - command: export ascii 'id.orig_h ni fe80:5074:1b53:7e7::/64'
  Expressions:
    tags: [server, expression]
    fixture: ServerTester
    steps:
      - command: import -b zeek
        input: data/zeek/conn.log.gz
      - command: export ascii 'fe80::5074:1b53:7e7:ad4d || 169.254.225.22'
      - command: export ascii '"OrfTtuI5G4e" || fe80::5074:1b53:7e7:ad4d'
  Node Type Query:
    tags: [node, type, ch6104]
    steps:
      - command: -N import -n 20 zeek
        input: data/zeek/conn.log.gz
      - command: -N export ascii '#type == "zeek.conn"'
  Type Query:
    tags: [server, type, ch5404]
    fixture: ServerTester
    steps:
      - command: import -b -n 20 zeek
        input: data/zeek/conn.log.gz
      - command: export ascii '#type == "zeek.conn"'
  Node json zeek conn:
    tags: [node, import-export, zeek, json]
    steps:
      - command: -N import json -s @./misc/schema/zeek-conn.schema -t zeek.conn
        input: data/json/conn.log.json.gz
      - command: -N export ascii 'duration > 6s'
      - command: -N export ascii '#timestamp >= 2011-08-15T03:48'
  Node suricata alert:
    tags: [node, import-export, suricata, eve]
    steps:
      - command: -N import suricata -r @./data/suricata/eve.json
      - command: -N export ascii 'src_ip == 172.16.17.32'
      - command: -N export csv '#type ~ /suricata.*/'

  Node argus csv:
    tags: [node, import-export, argus, csv]
    steps:
      - command: -N import csv -t argus.record
        input: data/csv/argus-M57-10k-pkts.csv.gz
      - command: -N export ascii 'State != "CON" && Dur > 4900ms'
      - command: -N export ascii 'Cause == "Status" && Dur > 1s'

  Manual 1:
    condition: 'import -h 2>&1 | grep -q pcap'
    tags: [examples]
    fixture: ServerTester
    steps:
      - command: import -b pcap
        input: data/pcap/example.pcap.gz
      - command: export ascii ':addr in 192.168.168.0/24'
      - command: export pcap 'sport > 60000/tcp && src !in 192.168.168.0/8'

  Manual 1 node:
    condition: 'import -h 2>&1 | grep -q pcap'
    tags: [examples]
    steps:
      - command: -N import pcap
        input: data/pcap/example.pcap.gz
      - command: -N export ascii ':addr in 192.168.168.0/24'
      - command: -N export pcap 'sport > 60000/tcp && src !in 192.168.168.0/8'
      - command: -N status
        transformation: jq '.node.index.statistics'

  Manual 2:
    tags: [examples, disabled]
    fixture: BiTopologyTester
    steps:
      - command: peer 'localhost:42124'
      - command: status

  Importing VLAN traffic:
    condition: 'import -h 2>&1 | grep -q pcap'
    tags: [import, pcap]
    steps:
      - command: -N import pcap
        input: data/pcap/vlan.pcap.gz
      - command: -N status
        transformation: jq '.node.index.statistics'

  Multi addr query:
    tags: [import-export, zeek]
    steps:
      - command: -N import zeek
        input: data/zeek/conn.log.gz
      - command: -N export ascii
        input: queries/multi_addr.txt

  Pivot:
    tags: [pivot]
    steps:
      - command: -N import suricata
        input: data/pcap/suricata/eve.json.gz
      - command: -N import zeek
        input: data/pcap/zeek/conn.log.gz
      - command: -N import zeek
        input: data/pcap/zeek/files.log.gz
      - command: -N import zeek
        input: data/pcap/zeek/http.log.gz
      - command: -N import zeek
        input: data/pcap/zeek/packet_filter.log.gz
      - command: -N import zeek
        input: data/pcap/zeek/ssh.log.gz
      - command: -N pivot zeek.conn '#type ~ /suricata.*/ && dest_ip == 192.168.127.12'
        transformation: awk '!/^#(open|close)/'
      - command: -N pivot zeek.conn '#type ~ /zeek.http/ && method == "GET"'
        transformation: awk '!/^#(open|close)/'
      - guard: 'import -h 2>&1 | grep -q pcap'
      - command: -N import pcap
        input: data/pcap/example.pcap.gz
      - command: -N pivot pcap.packet 'dest_ip == 192.168.127.12'
        transformation: tcpdump -n -tt -r -
  Arrow:
    condition: 'export -h 2>&1 | grep -q arrow'
    tags: [export, arrow]
    steps:
      - command: -N import zeek
        input: data/zeek/conn.log.gz
      - command: -N export -n 10 arrow '#type == "zeek.conn"'
        transformation: python @./misc/scripts/print-arrow.py
  Type Registry:
    tags: [import]
    steps:
      - command: -N import -n 1 test
      - command: -N status
        transformation: jq '.node."type-registry".statistics'


{'fraction_non_alphanumeric': 0.14038602492059615, 'fraction_numerical': 0.046518446127534814, 'mean_word_length': 2.055995221741078, 'pattern_counts': {'":': 4, '<': 11, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 58, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2240041', 'n_tokens_mistral': 7952, 'n_tokens_neox': 7189, 'n_words': 1923}