<commit_before>#!/usr/bin/perl -w
#
#      $Id$
#
#########################################################################
#									#
#			   Copyright (C)  2002				#
#	     			Internet2				#
#			   All Rights Reserved				#
#									#
#########################################################################
#
#	File:		collector.pl
#
#	Author:		<NAME>
#			Internet2
#
#	Date:		Tue Oct 08 11:15:49 MDT 2002
#
#	Description:	
#
#	Usage:
#
#	Environment:
#
#	Files:
#
#	Options:
use strict;
use FindBin;
use lib ("$FindBin::Bin");
use Getopt::Std;
use Socket;
use POSIX;
use File::Path;
use Digest::MD5;
use OWP;
use OWP::Syslog;
use OWP::RawIO;
use OWP::Archive;
use OWP::Utils;
use OWP::Digest;
use Sys::Syslog;
use File::Basename;
use Fcntl ':flock';
use FileHandle;
use IO::Socket;
use DB_File;
use Carp qw(cluck);

my @SAVEARGV = @ARGV;

my %options = (
	CONFDIR		=>	"c:",
	FOREGROUND	=>	"f",
	);
my %optnames = (
	c	=> "CONFDIR",
	f	=> "FOREGROUND",
	);
my %defaults = (
	CONFDIR	=> "$FindBin::Bin",
	);

my $options = join '', values %options;
my %setopts;
getopts($options,\%setopts);
foreach (keys %optnames){
	$defaults{$optnames{$_}} = $setopts{$_} if(defined($setopts{$_}));
}

# Don't daemonize any re-exec'd children
push @SAVEARGV, "-f" if(!defined($setopts{'f'}));

# Fetch configuration options.
my $conf = new OWP::Conf(%defaults);

local(*MYLOG);
# setup syslog
my $slog = tie *MYLOG, 'OWP::Syslog',
		facility	=> $conf->must_get_val(ATTR=>'SyslogFacility'),
		log_opts	=> 'pid',
		setlogsock	=> 'unix';
# make die/warn goto syslog, and also to STDERR.
$slog->HandleDieWarn(*STDERR);
undef $slog;	# Don't need the ref anymore, and untie won't work if
		# I keep the ref.

#
# Globals...
#
my $debug = $conf->get_val(ATTR=>'DEBUG');
my $devnull = $conf->must_get_val(ATTR=>"devnull");

#
# Initialize Mesh info
#
my @mtypes = $conf->must_get_val(ATTR=>'MESHTYPES');
my @nodes = $conf->must_get_val(ATTR=>'MESHNODES');

#
# Initialize digesting info
#
my @reslist = sort $conf->must_get_val(ATTR=>'DIGESTRESLIST');
my $digestcmd = $conf->must_get_val(ATTR=>'OWPBinDir');
$digestcmd .= "/";
$digestcmd .= $conf->must_get_val(ATTR=>'digestcmd');
my $sessionsuffix = $conf->must_get_val(ATTR=>'SessionSuffix');
my $digestsuffix = $conf->must_get_val(ATTR=>'DigestSuffix');

#
# data path information
#
my $datadir = $conf->must_get_val(ATTR=>'CentralDataDir');

#
# archive setup
#
my $archive = OWP::Archive->new(DATADIR=>$datadir);

#
# deamon values
#
my $foreground = $conf->get_val(ATTR=>'FOREGROUND');
my $serverport = $conf->must_get_val(ATTR=>'CentralHostPort');
my $dbfile = $conf->must_get_val(ATTR=>'CentralPerDirDB');
my $vtimefile = $conf->must_get_val(ATTR=>'CentralPerDirValidFile');
my $uptimedb = $datadir . "/" . $conf->must_get_val(ATTR=>'UpTimeDBFile');
my $udpmsglen = $conf->must_get_val(ATTR=>'UpTimeMaxMesgLen') + 0;
my $uptimeport = $conf->must_get_val(ATTR=>'UpTimeSendToPort');

my $timeout = $conf->must_get_val(ATTR=>'CentralHostTimeout');

#
# Build directories if needed.
#
my($mtype,$recv,$send,$raddr,$saddr,$res);
my %nodeupdate;	# hash vals contain list of dirs to update for each "node".
my @dirlist; # list of directories that must exist.
foreach $mtype (@mtypes){
	foreach $recv (@nodes){
		next if(!($raddr=$conf->get_val(NODE=>$recv,
						TYPE=>$mtype,
						ATTR=>'ADDR')));
		foreach $send (@nodes){
			next if(!($saddr=$conf->get_val(NODE=>$send,
							TYPE=>$mtype,
							ATTR=>'ADDR')));
			push @{$nodeupdate{$send}},
				"$datadir/$mtype/$raddr/$saddr";

			foreach $res (@reslist){
				push @dirlist,
					"$datadir/$mtype/$raddr/$saddr/$res";
			}
		}
	}
}
die "No valid paths in mesh?" if(!@dirlist);
mkpath(\@dirlist,0,0775);

chdir $datadir || die "Unable to chdir to $datadir";

#
# setup server socket.
#
my $Server = IO::Socket::INET->new(
			LocalPort	=>	$serverport,
			Proto		=>	'tcp',
			Type		=>	SOCK_STREAM,
			ReuseAddr	=>	1,
			Reuse		=>	1,
			Timeout		=>	$timeout,
			Listen		=>	SOMAXCONN) or
		die "Unable to create server socket for sessiondata: $!";

if(!$debug && !$foreground){
	daemonize(PIDFILE => 'collector.pid') ||
		die "Unable to daemonize process";
	# Re-setup syslog
#	untie *MYLOG;
#	$slog = tie *MYLOG, 'OWP::Syslog',
#		facility	=> $conf->must_get_val(ATTR=>'SyslogFacility'),
#		log_opts	=> 'pid',
#		setlogsock	=> 'unix';
	# make die/warn goto syslog, and also to STDERR.
#	$slog->HandleDieWarn();
#	undef $slog;	# Don't need the ref anymore, and untie won't work if
			# I keep the ref.
}

my (%children);

# just in case...
use constant MAX_UPTIMES_RESTARTS => 5;
my $uppid = uptimes(%nodeupdate);
$children{$uppid} = 'uptimes';
my $uprestarts = 0;

my ($reset,$die,$sigchld) = (0,0,0);

sub catch{
	my ($signame) = @_;

	return if !defined $signame;

	if($signame =~ /HUP/){
		$reset++;
	}
	elsif($signame =~ /CHLD/){
		$sigchld++;
	}
	else{
		$die++;
	}
	#
	# If we are in an eval - die from here to make the function return
	# and not automatically restart: ie accept.<commit_msg>Removed commented out code.<commit_after>#!/usr/bin/perl -w
#
#      $Id$
#
#########################################################################
#									#
#			   Copyright (C)  2002				#
#	     			Internet2				#
#			   All Rights Reserved				#
#									#
#########################################################################
#
#	File:		collector.pl
#
#	Author:		<NAME>
#			Internet2
#
#	Date:		Tue Oct 08 11:15:49 MDT 2002
#
#	Description:	
#
#	Usage:
#
#	Environment:
#
#	Files:
#
#	Options:
use strict;
use FindBin;
use lib ("$FindBin::Bin");
use Getopt::Std;
use Socket;
use POSIX;
use File::Path;
use Digest::MD5;
use OWP;
use OWP::Syslog;
use OWP::RawIO;
use OWP::Archive;
use OWP::Utils;
use OWP::Digest;
use Sys::Syslog;
use File::Basename;
use Fcntl ':flock';
use FileHandle;
use IO::Socket;
use DB_File;
use Carp qw(cluck);

my @SAVEARGV = @ARGV;

my %options = (
	CONFDIR		=>	"c:",
	FOREGROUND	=>	"f",
	);
my %optnames = (
	c	=> "CONFDIR",
	f	=> "FOREGROUND",
	);
my %defaults = (
	CONFDIR	=> "$FindBin::Bin",
	);

my $options = join '', values %options;
my %setopts;
getopts($options,\%setopts);
foreach (keys %optnames){
	$defaults{$optnames{$_}} = $setopts{$_} if(defined($setopts{$_}));
}

# Don't daemonize any re-exec'd children
push @SAVEARGV, "-f" if(!defined($setopts{'f'}));

# Fetch configuration options.
my $conf = new OWP::Conf(%defaults);

local(*MYLOG);
# setup syslog
my $slog = tie *MYLOG, 'OWP::Syslog',
		facility	=> $conf->must_get_val(ATTR=>'SyslogFacility'),
		log_opts	=> 'pid',
		setlogsock	=> 'unix';
# make die/warn goto syslog, and also to STDERR.
$slog->HandleDieWarn(*STDERR);
undef $slog;	# Don't need the ref anymore, and untie won't work if
		# I keep the ref.

#
# Globals...
#
my $debug = $conf->get_val(ATTR=>'DEBUG');
my $devnull = $conf->must_get_val(ATTR=>"devnull");

#
# Initialize Mesh info
#
my @mtypes = $conf->must_get_val(ATTR=>'MESHTYPES');
my @nodes = $conf->must_get_val(ATTR=>'MESHNODES');

#
# Initialize digesting info
#
my @reslist = sort $conf->must_get_val(ATTR=>'DIGESTRESLIST');
my $digestcmd = $conf->must_get_val(ATTR=>'OWPBinDir');
$digestcmd .= "/";
$digestcmd .= $conf->must_get_val(ATTR=>'digestcmd');
my $sessionsuffix = $conf->must_get_val(ATTR=>'SessionSuffix');
my $digestsuffix = $conf->must_get_val(ATTR=>'DigestSuffix');

#
# data path information
#
my $datadir = $conf->must_get_val(ATTR=>'CentralDataDir');

#
# archive setup
#
my $archive = OWP::Archive->new(DATADIR=>$datadir);

#
# deamon values
#
my $foreground = $conf->get_val(ATTR=>'FOREGROUND');
my $serverport = $conf->must_get_val(ATTR=>'CentralHostPort');
my $dbfile = $conf->must_get_val(ATTR=>'CentralPerDirDB');
my $vtimefile = $conf->must_get_val(ATTR=>'CentralPerDirValidFile');
my $uptimedb = $datadir . "/" . $conf->must_get_val(ATTR=>'UpTimeDBFile');
my $udpmsglen = $conf->must_get_val(ATTR=>'UpTimeMaxMesgLen') + 0;
my $uptimeport = $conf->must_get_val(ATTR=>'UpTimeSendToPort');

my $timeout = $conf->must_get_val(ATTR=>'CentralHostTimeout');

#
# Build directories if needed.
#
my($mtype,$recv,$send,$raddr,$saddr,$res);
my %nodeupdate;	# hash vals contain list of dirs to update for each "node".
my @dirlist; # list of directories that must exist.
foreach $mtype (@mtypes){
	foreach $recv (@nodes){
		next if(!($raddr=$conf->get_val(NODE=>$recv,
						TYPE=>$mtype,
						ATTR=>'ADDR')));
		foreach $send (@nodes){
			next if(!($saddr=$conf->get_val(NODE=>$send,
							TYPE=>$mtype,
							ATTR=>'ADDR')));
			push @{$nodeupdate{$send}},
				"$datadir/$mtype/$raddr/$saddr";

			foreach $res (@reslist){
				push @dirlist,
					"$datadir/$mtype/$raddr/$saddr/$res";
			}
		}
	}
}
die "No valid paths in mesh?" if(!@dirlist);
mkpath(\@dirlist,0,0775);

chdir $datadir || die "Unable to chdir to $datadir";

#
# setup server socket.
#
my $Server = IO::Socket::INET->new(
			LocalPort	=>	$serverport,
			Proto		=>	'tcp',
			Type		=>	SOCK_STREAM,
			ReuseAddr	=>	1,
			Reuse		=>	1,
			Timeout		=>	$timeout,
			Listen		=>	SOMAXCONN) or
		die "Unable to create server socket for sessiondata: $!";

if(!$debug && !$foreground){
	daemonize(PIDFILE => 'collector.pid') ||
		die "Unable to daemonize process";
}

my (%children);

# just in case...
use constant MAX_UPTIMES_RESTARTS => 5;
my $uppid = uptimes(%nodeupdate);
$children{$uppid} = 'uptimes';
my $uprestarts = 0;

my ($reset,$die,$sigchld) = (0,0,0);

sub catch{
	my ($signame) = @_;

	return if !defined $signame;

	if($signame =~ /HUP/){
		$reset++;
	}
	elsif($signame =~ /CHLD/){
		$sigchld++;
	}
	else{
		$die++;
	}
	#
	# If we are in an eval - die from here to make the function return
	# and not automatically restart: ie accept.
{'fraction_non_alphanumeric': 0.21527129370266626, 'fraction_numerical': 0.006639692914202718, 'mean_word_length': 4.667254556143445, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 145, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3473324', 'n_tokens_mistral': 4077, 'n_tokens_neox': 3791, 'n_words': 833}