<commit_before>// author: InMon Corp.
// version: 0.3
// date: 4/3/2016
// description: sFlow-RT IX Metrics
// copyright: Copyright (c) 2015,2016 InMon Corp.

// TODO:
// 1. verify that observed VLAN matches vlan_id in members file
// 2. link metrics to influxDB
// 3. port metrics to influxDB

include(scriptdir() + '/inc/trend.js');

var influxHost = getSystemProperty("ix.influx.host");
var influxDB = getSystemProperty("ix.influx.db") || "ix";
var influxURL = influxHost ? "http://"+influxHost+":8086/write?db="+influxDB : null;

var syslogHost = getSystemProperty("ix.syslog.host");
var syslogPort = getSystemProperty("ix.syslog.port") || 514;
var facility = 16; // local0
var severity = 5;  // notice

function sendWarning(msg) {
  if(syslogHost) syslog(syslogHost,syslogPort,facility,severity,msg);
  else logWarning(JSON.stringify(msg));
}

var topology = storeGet('topology');
if(topology) setTopology(topology);

var members = storeGet('members') || {};

var macToMember = {};
var ipGroups = {};
var numMembers = 0;
var numMacs = 0;
function updateMemberInfo() {
  var memberToMac,memberToIP,member,name,macs,ips,conns,j,conn,vlan_list,k,vlan,mac;
  if(!members.version) return;
  if(!(members.version === "0.4" || members.version === "0.5")) return;
  if(!members.member_list) return;

  memberToMac = {};
  memberToIP = {};
  macToMember = {};
  numMembers = 0;
  numMacs = 0;
  for(i = 0; i < members.member_list.length; i++) {
    member = members.member_list[i];
    if(!member) continue;
    name = member.name;
    if(!name) continue;
    numMembers++;
    macs = [];
    ips = [];
    conns = member.connection_list;
    if(!conns) continue;
    for(j = 0; j < conns.length; j++) {
      conn = conns[j];
      vlan_list = conn.vlan_list;
      if(!vlan_list) continue;
      for(var k = 0; k < vlan_list.length; k++) {
        vlan = vlan_list[k];
        if(!vlan) continue;
        if(vlan.ipv4 && vlan.ipv4.address) ips.push(vlan.ipv4.address);
        if(vlan.ipv6 && vlan.ipv6.address) ips.push(vlan.ipv6.address);
        mac = vlan.mac_address;
        if(!mac) continue;
        mac = mac.replace(/:/g,'').toUpperCase();
        if('UNKNOWN' === mac) continue;
        macs.push(mac);
        macToMember[mac] = name;
        numMacs++;
      }
    }
    if(ips.length > 0) memberToIP[name] = ips;
    if(macs.length > 0) memberToMac[name] = macs;
  }
  setGroups('ix_member',memberToIP);
  setMap('ix_member',memberToMac);
}
updateMemberInfo();

var trend = new Trend(300,1);
var points;

var SEP = '_SEP_';
var T = 15;
var N = 10;

// metrics
setFlow('ix_bytes', {value:'bytes', t:T, fs: SEP, filter:'direction=ingress'});
setFlow('ix_frames', {value:'frames', t:T, fs:SEP, filter:'direction=ingress'});
setFlow('ix_src', {keys:'map:macsource:ix_member', value:'bytes', n:N, t:T, fs:SEP, filter:'direction=ingress'});
setFlow('ix_dst', {keys:'map:macdestination:ix_member', value:'bytes', n:N, t:T, fs:SEP, filter:'direction=ingress'});
setFlow('ix_pair', {keys:'map:macsource:ix_member,map:macdestination:ix_member', value:'bytes', n:20, t:T, fs:SEP, filter:'direction=ingress'});
setFlow('ix_protocol', {keys:'ethernetprotocol', value:'bytes', n:N, t:T, fs:SEP, filter:'direction=ingress'}); 
setFlow('ix_pktsize', {keys:'range:bytes:0:63,range:bytes:64:64,range:bytes:65:123,range:bytes:128:255,range:bytes:256:511,range:bytes:512:1023,range:bytes:1024:1517,range:bytes:1518:1518,range:bytes:1519', value:'frames', n:9, t:T, filter:'direction=ingress'});

// find member macs
setFlow('ix_ip4', {keys:'macsource,group:ipsource:ix_member',value:'bytes',log:true,flowStart:true, n:N, t:T, fs:SEP});
setFlow('ix_ip6', {keys:'macsource,group:ip6source:ix_member',value:'bytes',log:true,flowStart:true, n:N, t:T, fs:SEP});

// find BGP connections
setFlow('ix_bgp', {keys:'or:[map:macsource:ix_member]:[group:ipsource:ix_member]:[group:ip6source:ix_member],or:[map:macdestination:ix_member]:[group:ipdestination:ix_member]:[group:ip6destination:ix_member]',value:'frames',filter:'tcpsourceport=179|tcpdestinationport=179',log:true,flowStart:true, n:N, t:T, fs:SEP});

// exceptions
setFlow('ix_srcmacunknown', {keys:'macsource', value:'bytes', filter:'direction=ingress&map:macsource:ix_member=null', n:N, t:T, fs:SEP});
setFlow('ix_dstmacunknown', {keys:'macdestination', value:'bytes', filter:'direction=ingress&map:macdestination:ix_member=null', n:N, t:T, fs:SEP});
setFlow('ix_badprotocol', {keys:'macsource,ethernetprotocol', value:'frames', filter:'ethernetprotocol!=2048,2054,34525', n:N, t:T, fs:SEP, log:true, flowStart:true});
<commit_msg>Fixed range error in packet size distribution<commit_after>// author: InMon Corp.
// version: 0.3
// date: 4/3/2016
// description: sFlow-RT IX Metrics
// copyright: Copyright (c) 2015,2016 InMon Corp.

// TODO:
// 1. verify that observed VLAN matches vlan_id in members file
// 2. link metrics to influxDB
// 3. port metrics to influxDB

include(scriptdir() + '/inc/trend.js');

var influxHost = getSystemProperty("ix.influx.host");
var influxDB = getSystemProperty("ix.influx.db") || "ix";
var influxURL = influxHost ? "http://"+influxHost+":8086/write?db="+influxDB : null;

var syslogHost = getSystemProperty("ix.syslog.host");
var syslogPort = getSystemProperty("ix.syslog.port") || 514;
var facility = 16; // local0
var severity = 5;  // notice

function sendWarning(msg) {
  if(syslogHost) syslog(syslogHost,syslogPort,facility,severity,msg);
  else logWarning(JSON.stringify(msg));
}

var topology = storeGet('topology');
if(topology) setTopology(topology);

var members = storeGet('members') || {};

var macToMember = {};
var ipGroups = {};
var numMembers = 0;
var numMacs = 0;
function updateMemberInfo() {
  var memberToMac,memberToIP,member,name,macs,ips,conns,j,conn,vlan_list,k,vlan,mac;
  if(!members.version) return;
  if(!(members.version === "0.4" || members.version === "0.5")) return;
  if(!members.member_list) return;

  memberToMac = {};
  memberToIP = {};
  macToMember = {};
  numMembers = 0;
  numMacs = 0;
  for(i = 0; i < members.member_list.length; i++) {
    member = members.member_list[i];
    if(!member) continue;
    name = member.name;
    if(!name) continue;
    numMembers++;
    macs = [];
    ips = [];
    conns = member.connection_list;
    if(!conns) continue;
    for(j = 0; j < conns.length; j++) {
      conn = conns[j];
      vlan_list = conn.vlan_list;
      if(!vlan_list) continue;
      for(var k = 0; k < vlan_list.length; k++) {
        vlan = vlan_list[k];
        if(!vlan) continue;
        if(vlan.ipv4 && vlan.ipv4.address) ips.push(vlan.ipv4.address);
        if(vlan.ipv6 && vlan.ipv6.address) ips.push(vlan.ipv6.address);
        mac = vlan.mac_address;
        if(!mac) continue;
        mac = mac.replace(/:/g,'').toUpperCase();
        if('UNKNOWN' === mac) continue;
        macs.push(mac);
        macToMember[mac] = name;
        numMacs++;
      }
    }
    if(ips.length > 0) memberToIP[name] = ips;
    if(macs.length > 0) memberToMac[name] = macs;
  }
  setGroups('ix_member',memberToIP);
  setMap('ix_member',memberToMac);
}
updateMemberInfo();

var trend = new Trend(300,1);
var points;

var SEP = '_SEP_';
var T = 15;
var N = 10;

// metrics
setFlow('ix_bytes', {value:'bytes', t:T, fs: SEP, filter:'direction=ingress'});
setFlow('ix_frames', {value:'frames', t:T, fs:SEP, filter:'direction=ingress'});
setFlow('ix_src', {keys:'map:macsource:ix_member', value:'bytes', n:N, t:T, fs:SEP, filter:'direction=ingress'});
setFlow('ix_dst', {keys:'map:macdestination:ix_member', value:'bytes', n:N, t:T, fs:SEP, filter:'direction=ingress'});
setFlow('ix_pair', {keys:'map:macsource:ix_member,map:macdestination:ix_member', value:'bytes', n:20, t:T, fs:SEP, filter:'direction=ingress'});
setFlow('ix_protocol', {keys:'ethernetprotocol', value:'bytes', n:N, t:T, fs:SEP, filter:'direction=ingress'}); 
setFlow('ix_pktsize', {keys:'range:bytes:0:63,range:bytes:64:64,range:bytes:65:127,range:bytes:128:255,range:bytes:256:511,range:bytes:512:1023,range:bytes:1024:1517,range:bytes:1518:1518,range:bytes:1519', value:'frames', n:9, t:T, filter:'direction=ingress'});

// find member macs
setFlow('ix_ip4', {keys:'macsource,group:ipsource:ix_member',value:'bytes',log:true,flowStart:true, n:N, t:T, fs:SEP});
setFlow('ix_ip6', {keys:'macsource,group:ip6source:ix_member',value:'bytes',log:true,flowStart:true, n:N, t:T, fs:SEP});

// find BGP connections
setFlow('ix_bgp', {keys:'or:[map:macsource:ix_member]:[group:ipsource:ix_member]:[group:ip6source:ix_member],or:[map:macdestination:ix_member]:[group:ipdestination:ix_member]:[group:ip6destination:ix_member]',value:'frames',filter:'tcpsourceport=179|tcpdestinationport=179',log:true,flowStart:true, n:N, t:T, fs:SEP});

// exceptions
setFlow('ix_srcmacunknown', {keys:'macsource', value:'bytes', filter:'direction=ingress&map:macsource:ix_member=null', n:N, t:T, fs:SEP});
setFlow('ix_dstmacunknown', {keys:'macdestination', value:'bytes', filter:'direction=ingress&map:macdestination:ix_member=null', n:N, t:T, fs:SEP});
setFlow('ix_badprotocol', {keys:'macsource,ethernetprotocol', value:'frames', filter:'ethernetprotocol!=2048,2054,34525', n:N, t:T, fs:SEP, log:true, flowStart:true});

{'fraction_non_alphanumeric': 0.1875, 'fraction_numerical': 0.029398954703832754, 'mean_word_length': 5.7092768444119795, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3903978', 'n_tokens_mistral': 3559, 'n_tokens_neox': 3380, 'n_words': 584}