<commit_before>#!/usr/bin/perl
use warnings;
use strict;
use IO::Socket;
use Time::HiRes qw(usleep nanosleep);

my $payload;
my $content;
my $ascii;

my $subexp;
my @payload_data;
my @payload_peices;
my $peice;
my $payloads;
my $peices;


open IN, 'rules.download' or die "The file has to actually exist, try again $!\n";	#input filehandle is IN


#This loop catalogues all content/pcre pecies for each rule into the @payload 2 diminsional array [rule][content/pcre part]
while (<IN>) {
	print "line $.\n";
	my $line = $_;
	print "$line\n";
	$peice = 0;	
	while (($line =~ /;\s*?((content|pcre|uricontent):!?\s?".+?";.+?)(content|pcre|uricontent).+\)$/) || ($line =~ /;\s*?((content|pcre|uricontent):!?\s?".+?".+)\)$/) ) {
		#$1 will capture first content:""; and all options after it (up to the next content or pcre)
		$subexp = $1;
		$payload_data[$.][$peice] = $subexp;
		$line =~ s/\Q$subexp\E//;
		$peice++;
	}
}

$payloads = @payload_data;	#get number of rules

#iterate through all pcre/content peices
my $i = 0;											#init the iterators
my $j = 0;											#init the iterators
while ($i < $payloads) {						#while we still have payloads
	$j = 0;
	while ($payload_data[$i][$j]) {				#while there are still content/pcre/uridata elements
		#if the element has an offset modifier, parse and capture it's value
		if (($payload_data[$i][$j]) && ($payload_data[$i][$j] =~ /;\s+offset:\s+(\d+)[^;]*?;/)) {
			$payload_peices[$i][$j][0] = $1;}
		#if the element has a distance modifier, parse and capture it's value
		if (($payload_data[$i][$j]) && ($payload_data[$i][$j] =~ /;\s+distance:\s+(\d+)[^;]*?;/)) {
			$payload_peices[$i][$j][1] = $1;}
		#if the element has an http_client_body modifier, parse and capture it's value
		if (($payload_data[$i][$j]) && ($payload_data[$i][$j] =~ /;\s+http_client_body[^;]*?;/)) {
			$payload_peices[$i][$j][2] = 'yes';
		} else {$payload_peices[$i][$j][2] = 'no'}
		#if the element has an http_cookie modifier, parse and capture it's value
		if (($payload_data[$i][$j]) && ($payload_data[$i][$j] =~ /;\s+http_cookie[^;]*?;/)) {
			$payload_peices[$i][$j][3] = 'yes';
		} else {$payload_peices[$i][$j][3] = 'no'}		
		#if the element has an http_header modifier, parse and capture it's value
		if (($payload_data[$i][$j]) && ($payload_data[$i][$j] =~ /;\s+http_header[^;]*?;/)) {
			$payload_peices[$i][$j][4] = 'yes';
		} else {$payload_peices[$i][$j][4] = 'no'}		
		#if the element has an http_method modifier, parse and capture it's value
		if (($payload_data[$i][$j]) && ($payload_data[$i][$j] =~ /;\s+http_method[^;]*?;/)) {
			$payload_peices[$i][$j][5] = 'yes';
		} else {$payload_peices[$i][$j][5] = 'no'}		
		#if the element has an http_uri modifier, parse and capture it's value
		if (($payload_data[$i][$j]) && ($payload_data[$i][$j] =~ /;\s+http_uri[^;]*?;/)) {
			$payload_peices[$i][$j][6] = 'yes';
		} else {$payload_peices[$i][$j][6] = 'no'}	
		#if the element has an http_stat_code modifier, parse and capture it's value
		if (($payload_data[$i][$j]) && ($payload_data[$i][$j] =~ /;\s+http_stat_code[^;]*?;/)) {
			$payload_peices[$i][$j][7] = 'yes';
		} else {$payload_peices[$i][$j][7] = 'no'}		
		#if the element has an http_stat_msg modifier, parse and capture it's value
		if (($payload_data[$i][$j]) && ($payload_data[$i][$j] =~ /;\s+http_stat_msg[^;]*?;/)) {
			$payload_peices[$i][$j][8] = 'yes';
		} else {$payload_peices[$i][$j][8] = 'no'}		
		#if the element has an isdataat modifier, parse and capture it's value
		if (($payload_data[$i][$j]) && ($payload_data[$i][$j] =~ /;\s+isdataat:\s+(\d+)[^;]*?;/)) {
			$payload_peices[$i][$j][9] = $1;}
		$j++;
	}
	$i++;
}

print "payload 153[0]: $payload_data[153][0]\n";
print "payload 153[0]'s http_uri: $payload_peices[153][0][9]\n";

=put

@payload_data datastructure:
$payload_data[rule][content or pcre part]

@payload_peices datastructure:
$peices[rule][content or pcre part][modifier]
	[0] = offset (value)<commit_msg>Update 8ball.pl<commit_after>#!/usr/bin/perl
use warnings;
use strict;
use IO::Socket;
use Time::HiRes qw(usleep nanosleep);

my $payload;
my $content;
my $ascii;

my $subexp;
my @payload_data;
my @payload_peices;
my @payload_metadata;
my $peice;
my $payloads;
my $peices;


open IN, 'rules.download' or die "The file has to actually exist, try again $!\n";	#input filehandle is IN


#This loop catalogues all content/pcre pecies for each rule into the @payload 2 diminsional array [rule][content/pcre part]
while (<IN>) {
#	print "line $.\n";
	my $line = $_;
#	print "$line\n";
	$peice = 0;	
	while (($line =~ /;\s*?((content|pcre|uricontent):!?\s?".+?";.+?)(content|pcre|uricontent).+\)$/) || ($line =~ /;\s*?((content|pcre|uricontent):!?\s?".+?".+)\)$/) ) {
		#$1 will capture first content:""; and all options after it (up to the next content or pcre)
		$subexp = $1;
		$payload_data[$.][$peice] = $subexp;
		$line =~ s/\Q$subexp\E//;
		$peice++;
	}
}
close IN;

#Get metadata/socket for each rule (which network/port to network/port), and other tidbits
open IN, 'rules.download' or die "The file has to actually exist, try again $!\n";	#input filehandle is IN
while (<IN>) {
	my $line = $_;
	if ($line =~ /^#*?alert\s+([^\s]+?)\s+([^\s]+?)\s+([^\s]+?)\s+.+?>\s+([^\s]+?)\s+([^\s]+?)\s+/) {
		$payload_metadata[$.][0] = $1;
		$payload_metadata[$.][1] = $2;
		$payload_metadata[$.][2] = $3;
		$payload_metadata[$.][3] = $4;
		$payload_metadata[$.][4] = $5;		
	}
}
close IN;

$payloads = @payload_data;	#get number of rules

#iterate through all pcre/content peices
my $i = 0;											#init the iterators
my $j = 0;											#init the iterators
while ($i < $payloads) {						#while we still have payloads
	$j = 0;
	while ($payload_data[$i][$j]) {				#while there are still content/pcre/uridata elements
		#if the element has an offset modifier, parse and capture it's value
		if (($payload_data[$i][$j]) && ($payload_data[$i][$j] =~ /;\s+offset:\s+(\d+)[^;]*?;/)) {
			$payload_peices[$i][$j][0] = $1;}
		#if the element has a distance modifier, parse and capture it's value
		if (($payload_data[$i][$j]) && ($payload_data[$i][$j] =~ /;\s+distance:\s+(\d+)[^;]*?;/)) {
			$payload_peices[$i][$j][1] = $1;}
		#if the element has an http_client_body modifier, parse and capture it's value
		if (($payload_data[$i][$j]) && ($payload_data[$i][$j] =~ /;\s+http_client_body[^;]*?;/)) {
			$payload_peices[$i][$j][2] = 'yes';
		} else {$payload_peices[$i][$j][2] = 'no'}
		#if the element has an http_cookie modifier, parse and capture it's value
		if (($payload_data[$i][$j]) && ($payload_data[$i][$j] =~ /;\s+http_cookie[^;]*?;/)) {
			$payload_peices[$i][$j][3] = 'yes';
		} else {$payload_peices[$i][$j][3] = 'no'}		
		#if the element has an http_header modifier, parse and capture it's value
		if (($payload_data[$i][$j]) && ($payload_data[$i][$j] =~ /;\s+http_header[^;]*?;/)) {
			$payload_peices[$i][$j][4] = 'yes';
		} else {$payload_peices[$i][$j][4] = 'no'}		
		#if the element has an http_method modifier, parse and capture it's value
		if (($payload_data[$i][$j]) && ($payload_data[$i][$j] =~ /;\s+http_method[^;]*?;/)) {
			$payload_peices[$i][$j][5] = 'yes';
		} else {$payload_peices[$i][$j][5] = 'no'}		
		#if the element has an http_uri modifier, parse and capture it's value
		if (($payload_data[$i][$j]) && ($payload_data[$i][$j] =~ /;\s+http_uri[^;]*?;/)) {
			$payload_peices[$i][$j][6] = 'yes';
		} else {$payload_peices[$i][$j][6] = 'no'}	
		#if the element has an http_stat_code modifier, parse and capture it's value
		if (($payload_data[$i][$j]) && ($payload_data[$i][$j] =~ /;\s+http_stat_code[^;]*?;/)) {
			$payload_peices[$i][$j][7] = 'yes';
		} else {$payload_peices[$i][$j][7] = 'no'}		
		#if the element has an http_stat_msg modifier, parse and capture it's value
		if (($payload_data[$i][$j]) && ($payload_data[$i][$j] =~ /;\s+http_stat_msg[^;]*?;/)) {
			$payload_peices[$i][$j][8] = 'yes';
		} else {$payload_peices[$i][$j][8] = 'no'}		
		#if the element has an isdataat modifier, parse and capture it's value
		if (($payload_data[$i][$j]) && ($payload_data[$i][$j] =~ /;\s+isdataat:\s+(\d+)[^;]*?;/)) {
			$payload_peices[$i][$j][9] = $1;}
		$j++;
	}
	$i++;
}

print "payload 153[0]: $payload_data[153][0]\n";
print "payload 153[0]'s http_uri: $payload_peices[153][0][9]\n";
print "payload metadata\n";
print "\tprotocol: $payload_metadata[153][0]\n";
print "\tsource net: $payload_metadata[153][1]\n";
print "\tsource port: $payload_metadata[153][2]\n";
print "\tdest net: $payload_metadata[153][3]\n";
print "\tdest port: $payload_metadata[153][4]\n";

=put

@payload_data datastructure:
$payload_data[rule][content or pcre part]

@payload_peices datastructure:
$peices[rule][content or pcre part][modifier]
	[0] = offset (value)
{'fraction_non_alphanumeric': 0.24253559944878272, 'fraction_numerical': 0.013895268718419843, 'mean_word_length': 4.790558510638298, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4677157', 'n_tokens_mistral': 3780, 'n_tokens_neox': 3479, 'n_words': 821}