<commit_before>#!/usr/bin/perl -w
#
# $Id$
#
# written by <NAME> <<EMAIL>> for Monzoon Networks AG
# cli params/rrd storage/sampling mods Steve Colam <<EMAIL>>

use strict;
use 5.010;
use IO::Select;
use IO::Socket;
use RRDs;
use Getopt::Std;
use Scalar::Util qw(looks_like_number);

my %knownlinks;
my %link_samplingrates;

my $ascache = {};
my $ascache_lastflush = 0;
my $ascache_flush_interval = 25;
my $ascache_flush_number = 0;

my $MAXREAD = 8192;
my $childrunning = 0;

# NetFlow
my $server_port = 9000;
my $v5_header_len = 24;
my $v5_flowrec_len = 48;
my $v8_header_len = 28;
my $v8_flowrec_len = 28;
my $v9_header_len = 20;
my $v9_templates = {};
my $v10_header_len = 16;
my $v10_templates = {};

# sFlow
my $sflow_server_port = 6343;

use vars qw/ %opt /;
getopts('r:p:P:k:a:n', \%opt);

my $usage = "$0 [-rpPka]\n".
	"\t-r <path to RRD files>\n".
	"\t(-p <NetFlow UDP listen port - default $server_port, use 0 to disable NetFlow)\n".
	"\t(-P <sFlow UDP listen port - default $sflow_server_port, use 0 to disable sFlow)\n".
	"\t-k <path to known links file>\n".
	"\t-a <your own AS number> - only required for sFlow\n".
	"\t-n enable peer-as statistics\n";

my $rrdpath = $opt{'r'};
my $knownlinksfile = $opt{'k'};
my $myas_opt = $opt{'a'};
my $peerasstats = $opt{'n'};

die("$usage") if (!defined($rrdpath) || !defined($knownlinksfile));

die("$rrdpath does not exist or is not a directory\n") if ! -d $rrdpath;
die("$knownlinksfile does not exist or is not a file\n") if ! -f $knownlinksfile;

if (defined($opt{'p'})) {
	$server_port = $opt{'p'};
	die("NetFlow server port is non numeric\n") if $server_port !~ /^[0-9]+$/;
}

if (defined($opt{'P'})) {
	$sflow_server_port = $opt{'P'};
	die("sFlow server port is non numeric\n") if $sflow_server_port !~ /^[0-9]+$/;
}

if ($sflow_server_port == $server_port) {
	die("sFlow server port can't be the same as NetFlow server port\n");
}

my %myas;
if($sflow_server_port > 0){
	die('No ASN found, please specify -a') if !defined($myas_opt);
	%myas = map {$_ => 1 } split(',', $myas_opt);
	for my $i (%myas){
		next if !defined($i);
		die("Your AS number is non numeric ($i)\n") if ($i !~ /^[0-9]+$/);
	}
}

if (!$sflow_server_port && $peerasstats) {
    die("peer-as statistics only work with sFlow\n");
}


# reap dead children
$SIG{CHLD} = \&REAPER;
$SIG{TERM} = \&TERM;
$SIG{INT} = \&TERM;
$SIG{HUP} = \&read_knownlinks;

sub REAPER {
	wait;
	$childrunning = 0;
	$SIG{CHLD} = \&REAPER;
}

sub TERM {
	print "SIGTERM received\n";
	flush_cache(1);
	while (wait() != -1) {}
	exit 0;
}

# read known links file
read_knownlinks();

my ($lsn_nflow, $lsn_sflow);
my $sel = IO::Select->new();

# prepare to listen for NetFlow UDP packets
if ($server_port > 0) {
	$lsn_nflow = IO::Socket::INET->new(LocalPort => $server_port, Proto => "udp")
		or die "Couldn't be a NetFlow UDP server on port $server_port : $@\n";
	$sel->add($lsn_nflow);
}
# prepare to listen for sFlow UDP packets
if ($sflow_server_port > 0) {
	require Net::sFlow;
	$lsn_sflow = IO::Socket::INET->new(LocalPort => $sflow_server_port, Proto => "udp")
		or die "Couldn't be a sFlow UDP server on port $sflow_server_port : $@\n";
	$sel->add($lsn_sflow);
}

my ($him,$datagram,$flags);

# main datagram receive loop
while (1) {
	while (my @ready = $sel->can_read) {
		foreach my $server (@ready) {
			$him = $server->recv($datagram, $MAXREAD);
			next if (!$him);
			
			my ($port, $ipaddr) = sockaddr_in($server->peername);
			
			if (defined($lsn_nflow) && $server == $lsn_nflow) {
				my ($version) = unpack("n", $datagram);
				
				if ($version == 5) {
					parse_netflow_v5($datagram, $ipaddr);
				} elsif ($version == 8) {
					parse_netflow_v8($datagram, $ipaddr);
				} elsif ($version == 9) {
					parse_netflow_v9($datagram, $ipaddr);
				} elsif ($version == 10) {
					parse_netflow_v10($datagram, $ipaddr);
				} else {
					print "unknown NetFlow version: $version\n";
				}
			}
			elsif (defined($lsn_sflow) && $server == $lsn_sflow) {
				parse_sflow($datagram, $ipaddr);
			}
		}
	}
}

sub parse_netflow_v5 {
	my $datagram = shift;
	my $ipaddr = shift;
	
	my ($version, $count, $sysuptime, $unix_secs, $unix_nsecs,
	  $flow_sequence, $engine_type, $engine_id, $aggregation,
	  $agg_version) = unpack("nnNNNNCCCC", $datagram);

	my $flowrecs = substr($datagram, $v5_header_len);
	
	for (my $i = 0; $i < $count; $i++) {
		my $flowrec = substr($datagram, $v5_header_len + ($i*$v5_flowrec_len), $v5_flowrec_len);
		my @flowdata = unpack("NNNnnNNNNnnccccnnccN", $flowrec);
		#print "ipaddr: " . inet_ntoa($ipaddr) . " octets: $flowdata[6] srcas: $flowdata[15] dstas: $flowdata[16] in: $flowdata[3] out: $flowdata[4] 4 \n";
		handleflow($ipaddr, $flowdata[6], $flowdata[15], $flowdata[16], $flowdata[3], $flowdata[4], 4, 'netflow');
	}
}

sub parse_netflow_v8 {<commit_msg>Support optional IP to ASN mapping for netflow v5<commit_after>#!/usr/bin/perl -w
#
# $Id$
#
# written by <NAME> <<EMAIL>> for Monzoon Networks AG
# cli params/rrd storage/sampling mods <NAME> <<EMAIL>>

use strict;
use 5.010;
use IO::Select;
use IO::Socket;
use RRDs;
use Getopt::Std;
use Scalar::Util qw(looks_like_number);
use ip2as;

my %knownlinks;
my %link_samplingrates;

my $ascache = {};
my $ascache_lastflush = 0;
my $ascache_flush_interval = 25;
my $ascache_flush_number = 0;

my $MAXREAD = 8192;
my $childrunning = 0;

# NetFlow
my $server_port = 9000;
my $v5_header_len = 24;
my $v5_flowrec_len = 48;
my $v8_header_len = 28;
my $v8_flowrec_len = 28;
my $v9_header_len = 20;
my $v9_templates = {};
my $v10_header_len = 16;
my $v10_templates = {};

# sFlow
my $sflow_server_port = 6343;

use vars qw/ %opt /;
getopts('r:p:P:k:a:nm:', \%opt);

my $usage = "$0 [-rpPka]\n".
	"\t-r <path to RRD files>\n".
	"\t(-p <NetFlow UDP listen port - default $server_port, use 0 to disable NetFlow)\n".
	"\t(-P <sFlow UDP listen port - default $sflow_server_port, use 0 to disable sFlow)\n".
	"\t-k <path to known links file>\n".
	"\t-a <your own AS number> - only required for sFlow\n".
	"\t-n enable peer-as statistics\n".
	"\t-m IP<->ASN mapping\n";

my $rrdpath = $opt{'r'};
my $knownlinksfile = $opt{'k'};
my $myas_opt = $opt{'a'};
my $peerasstats = $opt{'n'};
my $mapping = $opt{'m'};

die("$usage") if (!defined($rrdpath) || !defined($knownlinksfile));

die("$rrdpath does not exist or is not a directory\n") if ! -d $rrdpath;
die("$knownlinksfile does not exist or is not a file\n") if ! -f $knownlinksfile;

if (defined($opt{'p'})) {
	$server_port = $opt{'p'};
	die("NetFlow server port is non numeric\n") if $server_port !~ /^[0-9]+$/;
}

if (defined($opt{'P'})) {
	$sflow_server_port = $opt{'P'};
	die("sFlow server port is non numeric\n") if $sflow_server_port !~ /^[0-9]+$/;
}

if ($sflow_server_port == $server_port) {
	die("sFlow server port can't be the same as NetFlow server port\n");
}

my %myas;
if($sflow_server_port > 0){
	die('No ASN found, please specify -a') if !defined($myas_opt);
	%myas = map {$_ => 1 } split(',', $myas_opt);
	for my $i (%myas){
		next if !defined($i);
		die("Your AS number is non numeric ($i)\n") if ($i !~ /^[0-9]+$/);
	}
}

if (!$sflow_server_port && $peerasstats) {
    die("peer-as statistics only work with sFlow\n");
}


# reap dead children
$SIG{CHLD} = \&REAPER;
$SIG{TERM} = \&TERM;
$SIG{INT} = \&TERM;
$SIG{HUP} = \&read_knownlinks;

sub REAPER {
	wait;
	$childrunning = 0;
	$SIG{CHLD} = \&REAPER;
}

sub TERM {
	print "SIGTERM received\n";
	flush_cache(1);
	while (wait() != -1) {}
	exit 0;
}

# read known links file
read_knownlinks();

my ($lsn_nflow, $lsn_sflow);
my $sel = IO::Select->new();

# prepare to listen for NetFlow UDP packets
if ($server_port > 0) {
	$lsn_nflow = IO::Socket::INET->new(LocalPort => $server_port, Proto => "udp")
		or die "Couldn't be a NetFlow UDP server on port $server_port : $@\n";
	$sel->add($lsn_nflow);
}
# prepare to listen for sFlow UDP packets
if ($sflow_server_port > 0) {
	require Net::sFlow;
	$lsn_sflow = IO::Socket::INET->new(LocalPort => $sflow_server_port, Proto => "udp")
		or die "Couldn't be a sFlow UDP server on port $sflow_server_port : $@\n";
	$sel->add($lsn_sflow);
}

my ($him,$datagram,$flags);

if (defined($mapping)) {
	ip2as::init($mapping);
} else {
	#I don't use the mapping, to use an empty one
	ip2as::init('/dev/null');
}

# main datagram receive loop
while (1) {
	while (my @ready = $sel->can_read) {
		foreach my $server (@ready) {
			$him = $server->recv($datagram, $MAXREAD);
			next if (!$him);
			
			my ($port, $ipaddr) = sockaddr_in($server->peername);
			
			if (defined($lsn_nflow) && $server == $lsn_nflow) {
				my ($version) = unpack("n", $datagram);
				
				if ($version == 5) {
					parse_netflow_v5($datagram, $ipaddr);
				} elsif ($version == 8) {
					parse_netflow_v8($datagram, $ipaddr);
				} elsif ($version == 9) {
					parse_netflow_v9($datagram, $ipaddr);
				} elsif ($version == 10) {
					parse_netflow_v10($datagram, $ipaddr);
				} else {
					print "unknown NetFlow version: $version\n";
				}
			}
			elsif (defined($lsn_sflow) && $server == $lsn_sflow) {
				parse_sflow($datagram, $ipaddr);
			}
		}
	}
}

sub replace_asn {
	my $ip = shift;
	my $asn = shift;

	my $new_asn = ip2as::getas4ip($ip);
	if (defined($new_asn)) {
		return $new_asn;
	} else {
		return $asn;
	}
}

sub parse_netflow_v5 {
	my $datagram = shift;
	my $ipaddr = shift;
	
	my ($version, $count, $sysuptime, $unix_secs, $unix_nsecs,
	  $flow_sequence, $engine_type, $engine_id, $aggregation,
	  $agg_version) = unpack("nnNNNNCCCC", $datagram);

	my $flowrecs = substr($datagram, $v5_header_len);
	
	for (my $i = 0; $i < $count; $i++) {
		my $flowrec = substr($datagram, $v5_header_len + ($i*$v5_flowrec_len), $v5_flowrec_len);
		my @flowdata = unpack("NNNnnNNNNnnccccnnccN", $flowrec);
		my $srcip = join '.', unpack 'C4', pack 'N', $flowdata[0];
		my $dstip = join '.', unpack 'C4', pack 'N', $flowdata[1];

		my $srcas = replace_asn($srcip, $flowdata[15]);
		my $dstas = replace_asn($dstip, $flowdata[16]);

		#print "ipaddr: " . inet_ntoa($ipaddr) . " octets: $flowdata[6] srcas: $srcas dstas: $dstas in: $flowdata[3] out: $flowdata[4] 4 \n";
		handleflow($ipaddr, $flowdata[6], $srcas, $dstas, $flowdata[3], $flowdata[4], 4, 'netflow');
	}
}

sub parse_netflow_v8 {
{'fraction_non_alphanumeric': 0.18815570153936662, 'fraction_numerical': 0.019021472693401313, 'mean_word_length': 4.255023183925811, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4892070', 'n_tokens_mistral': 4544, 'n_tokens_neox': 4205, 'n_words': 1064}