<commit_before>#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2008 University of Utah and the Flux Group.
# All rights reserved.
#
# Kernel, jail, netstat, route, ifconfig, ipfw, header files.
#
use strict;
use English;
use Getopt::Std;
use Fcntl;
use IO::Handle;
use Socket;
use Fcntl ':flock';

# Drag in path stuff so we can find emulab stuff. Also untaints path.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }

use libsetup qw(REMOTE LOCALROOTFS TMTOPOMAP TMLTMAP TMLTPMAP
		TBDebugTimeStamp);
use libtmcc;

#
# Questions:
#

#
# Create a jailed environment. There are some stub files stored in
# /etc/jail that copied into the jail.
#
sub usage()
{
    print("Usage: mkvserver.pl [-V] [-s] [-i <ipaddr>] [-p <pid>] ".
	  "[-h <hostname>] <vnodeid>\n");
    exit(-1);
}
my  $optlist = "Vi:p:e:sh:";

#
# Only real root can run this script.
#
if ($UID) {
    die("Must be root to run this script!\n");
}

#
# Catch ^C and exit with error. 
#
my $leaveme = 0;
sub handler ($) {
    my ($signame) = @_;
    
    $SIG{INT}  = 'IGNORE';
    $SIG{USR1} = 'IGNORE';
    $SIG{TERM} = 'IGNORE';
    $SIG{HUP}  = 'IGNORE';

    if ($signame eq 'USR1') {
	$leaveme = 1;
    }
    fatal("Caught a SIG${signame}! Killing the vserver ...");
}
$SIG{INT}  = \&handler;
$SIG{USR1} = \&handler;
$SIG{HUP}  = \&handler;
$SIG{TERM} = 'IGNORE';

#
# Turn off line buffering on output
#
STDOUT->autoflush(1);
STDERR->autoflush(1);

# XXX
my $JAILCNET	   = "172.16.0.0";
my $JAILCNETMASK   = "255.240.0.0";

#
# Locals
#
my $JAILPATH	   = "/var/emulab/jails";
my $ETCVSERVER     = "/usr/local/etc/emulab/vserver";
my $VSERVER	   = "/usr/sbin/vserver";
my $TMCC	   = "$BINDIR/tmcc";
my $VSERVERDIR     = "/vservers";
my $JAILCONFIG     = "jailconfig";
my @ROOTCPDIRS     = ("etc", "root");
my @ROOTMKDIRS     = ("dev", "tmp", "var", "usr", "proc", "users", "lib",
		      "bin", "sbin", "home");
my @ROOTMNTDIRS    = ("bin", "sbin", "usr", "lib");
my @EMUVARDIRS	   = ("logs", "db", "jails", "boot", "lock");
my $IP;
my $IPMASK;
my $PID;
my $VDIR;
my $CDIR;
my $idnumber;
my $jailhostname;
my $jailpid;
my $tmccpid;
my $debug	   = 1;
my @mntpoints      = ();
my $USEVCNETROUTES = 0;
my @controlroutes  = ();
my $interactive    = 0;
my $cleaning	   = 0;

# This stuff is passed from tmcd, which we parse into a config string
# and an option set.
my %jailconfig  = ();
my $sshdport    = 50000;	# Bogus default, good for testing.
my $jailflags   = 0;
my @jailips     = ();		# List of jail IPs (for routing table).

#
# Protos
#
sub mkvserver($);
sub upvserver($);
sub LoopMount($$);
sub PreparePhysNode();
sub fatal($);
sub getjailconfig($);
sub setjailoptions();
sub mysystem($);
sub cleanup();

#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (@ARGV != 1) {
    usage();
}
my $vnodeid = $ARGV[0];

#
# Untaint the arguments.
#
if ($vnodeid =~ /^([-\w\/\.]+)$/) {
    $vnodeid = $1;
    $jailhostname = $1;
}
else {
    die("Tainted argument $vnodeid!\n");
}

if (defined($options{'s'})) {
    $interactive = 1;
}

TBDebugTimeStamp("mkjail starting to do real work");

#
# Get the parent IP.
# 
my $hostname = `hostname`;
my $hostip;

# Untaint and strip newline.
if ($hostname =~ /^([-\w\.]+)$/) {
    $hostname = $1;

    my (undef,undef,undef,undef,@ipaddrs) = gethostbyname($hostname);
    $hostip = inet_ntoa($ipaddrs[0]);
}

#
# If no IP, then it defaults to our hostname's IP, *if* none is provided
# by the jail configuration file. That check is later. 
# 
if (defined($options{'i'})) {
    $IP = $options{'i'};

    if ($IP =~ /^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/) {
	$IP = $1;
    }
    else {
	die("Tainted argument $IP!\n");
    }
}

if (defined($options{'p'})) {
    $PID = $options{'p'};

    if ($PID =~ /^([-\@\w]+)$/) {
	$PID = $1;
    }
    else {
	die("Tainted argument $PID.");
    }
}

if (defined($options{'h'})) {
    $jailhostname = $options{'h'};

    if ($jailhostname =~ /^([-\w\.]+)$/) {
	$jailhostname = $1;
    }
    else {
	die("Tainted argument $jailhostname.");
    }
}

#
# In most cases, the $vnodeid directory will have been created by the caller,
# and a config file possibly dropped in.
# When debugging, we have to create it here. 
# 
chdir($JAILPATH) or
    die("Could not chdir to $JAILPATH: $!\n");

if (! -e $vnodeid) {
    mkdir($vnodeid, 0770) or
	fatal("Could not mkdir $vnodeid in $JAILPATH: $!");
}
else {
    TBDebugTimeStamp("mkjail getting jail config");
    getjailconfig("$JAILPATH/$vnodeid");
}

my $phys_cnet_if = `control_interface`;
chomp($phys_cnet_if);

#
# See if special options supported, and if so setup args as directed.
#
TBDebugTimeStamp("mkjail setting jail options");
setjailoptions();

# Do some prep stuff on the physical node if this is the first vserver.
PreparePhysNode();

print("Setting up jail for $vnodeid using $IP\n")
    if ($debug);

$VDIR = "$VSERVERDIR/$vnodeid";
$CDIR = "/etc/vservers/$vnodeid";

#
# Create the vserver.
#
if (-e $VDIR) {
    #
    # Try to pick up where we left off.
    #
    TBDebugTimeStamp("mkjail restoring root fs");
    upvserver("$vnodeid");
}
else {
    #
    # Create the root filesystem.
    #
    TBDebugTimeStamp("Creating vserver");
    mkvserver("$vnodeid");
}
TBDebugTimeStamp("mkjail done with root fs");
startproxy("$vnodeid");

#
# Start the vserver. If all goes well, this will exit cleanly, with the
# vserver running in its new context. Still, lets protect it with a timer
# since it might get hung up inside and we do not want to get stuck here.
#
my $childpid = fork();
if ($childpid) {
    #local $SIG{ALRM} = sub { kill("TERM", $childpid); };
    #alarm 30;
    waitpid($childpid, 0);
    #alarm 0;

    #
    # If failure then cleanup.
    #
    if ($?) {
	fatal("mkvserver: $vnodeid vserver startup exited with $?");
    }
}
else {
    $SIG{TERM} = 'DEFAULT';

    TBDebugTimeStamp("mkvserver: starting the vserver");

    exec("$VSERVER $vnodeid start");
    die("*** $0:\n".
	"    exec failed to start the vserver!\n");
}

#
# If this file does not exist, the inner setup failed somehow. Stop now.
#
fatal("vserver did not appear to set up properly. Exiting ...")
    if (! -e "$VDIR/$BOOTDIR/vrunning");

print "vserver for $vnodeid started. Waiting ...\n";
$jailpid = fork();
if ($jailpid) {
    #
    # We do not really care about the exit status of the jail, we just want
    # to know when it dies inside.
    #
    while (1) {
	my $kidpid = waitpid(-1, 0);

	if ($kidpid == $jailpid) {
	    undef($jailpid);
	    last;
	}
	if ($kidpid == $tmccpid) {
	    print("TMCC proxy exited with status $?. Restarting ...\n");
	    startproxy("$vnodeid");
	    next;
	}
	print("Unknown child $kidpid exited with status $?!\n");
    }
}
else {
    $SIG{TERM} = 'DEFAULT';

    exec("$VSERVER $vnodeid exec sleep 1000000");
    die("*** $0:\n".
	"    exec failed to start the jail!\n");
}
print "vserver for $vnodeid has died. Cleaning up ...\n";
cleanup();
exit(0);

#
# Create a root filesystem for the vserver.
#
sub mkvserver($)
{
    my ($vnodeid) = @_;
    my $interface;

    if (defined($IP)) {
	system("/usr/lib/util-vserver/mask2prefix $JAILCNETMASK");
	my $prefix = $? >> 8;
	$interface = "${vnodeid}=${phys_cnet_if}:${IP}/${prefix}";
    }
    else {
	$interface = "nodev:0.0.0.0/0";
    }

    # Create the skeleton vserver. It will be mostly empty.
    mysystem("$VSERVER $vnodeid build --force -m skeleton ".
	     "--hostname $jailhostname --interface $interface ".
	     "--flags persistent");

    # The filesystem for the vserver lands here.
    my $vdir = $VDIR;
    # The configuration directory is here.
    my $cdir = $CDIR;

    #
    # Copy in the top level directories. 
    #
    foreach my $dir (@ROOTCPDIRS) {
	mysystem("rsync -a /$dir $vdir");
    }
    TBDebugTimeStamp("mkvserver: Copying root cp dirs done!");

    #
    # Set vserver "capabilities".
    #
    # Allows binding to TCP/UDP sockets below 1024
    mysystem("echo 'NET_BIND_SERVICE' > $cdir/bcapabilities");

    #
    # Clean out some stuff from /etc.
    #
    mysystem("/bin/rm -rf $vdir/etc/rc.d/rc*.d/*");

    #
    # Make some other directories that are need in /root.
    #
    foreach my $dir (@ROOTMKDIRS) {
	if (! -e "$vdir/$dir") {
	    mkdir("$vdir/$dir", 0755) or
		fatal("Could not mkdir '$dir' in $vdir: $!");
	}
    }
    TBDebugTimeStamp("mkvserver: Creating root mkdir dirs done!");

    #
    # Mount (read-only) these other directories to save space.
    #
    foreach my $dir (@ROOTMNTDIRS) {
	LoopMount("/$dir", "$vdir/$dir");
    }
    TBDebugTimeStamp("mkvserver: Mounting root dirs done!");

    # /tmp is special.
    mysystem("chmod 1777 $vdir/tmp");

    #
    # Duplicate the /var hierarchy without the contents.
    #
    open(VARDIRS, "find /var -type d -print |")
	or fatal("Could not start find on /var");
    while (<VARDIRS>) {
	my $dir = $_;
	chomp($dir);

	mysystem("rsync -dlptgoD $dir $vdir$dir");
    }
    close(VARDIRS);

    #
    # Get a list of all the plain files and create zero length versions
    # in the new var.
    #
    opendir(DIR, "/var/log") or
	fatal("Cannot opendir /var/log: $!");
    my @logs = grep { -f "/var/log/$_" } readdir(DIR);
    closedir(DIR);

    foreach my $log (@logs) {
	mysystem("touch $vdir/var/log/$log");
    }
    TBDebugTimeStamp("mkvserver: finished setting up /var");
    
    # Create a local logs directory. 
    mysystem("mkdir -p $vdir/local/logs")
	if (! -e "$vdir/local/logs");

    #
    # Now a bunch of stuff to set up a nice environment in the jail.
    #
    mysystem("ln -s ../../init.d/syslog $vdir/etc/rc3.d/S80syslog");
    mysystem("ln -s ../../init.d/sshd $vdir/etc/rc3.d/S85sshd");
    mysystem("ln -s ../../init.d/syslog $vdir/etc/rc6.d/K85syslog");
    mysystem("ln -s ../../init.d/sshd $vdir/etc/rc6.d/K80sshd");
    mysystem("cp -p $ETCVSERVER/rc.invserver $vdir/etc/rc3.d/S99invserver");
    mysystem("cp -p $ETCVSERVER/rc.invserver $vdir/etc/rc6.d/K99invserver");

    # Kill anything that uses /dev/console in syslog; will not work.
    mysystem("sed -i.bak ".
	     "-e '/console/d' ".
	     "-e '/users/d' ".
	     "$vdir/etc/syslog.conf");
    
    # Premunge the sshd config: no X11 forwarding and get rid of old
    # ListenAddresses.
    #
    mysystem("sed -i.bak ".
	     "-e 's/^X11Forwarding.*yes/X11Forwarding no/' ".
	     "-e '/^ListenAddress/d' ".
	     "$vdir/etc/ssh/sshd_config");
    
    # Port/Address for sshd.
    if ($IP ne $hostip) {
	# XXX This can come out (sshd bind to port 22) once we have
	# the network stack stuff in the kernel.
	mysystem("echo 'Port $sshdport' >> $vdir/etc/ssh/sshd_config");

	mysystem("echo 'ListenAddress $IP' >> ".
		 "$vdir/etc/ssh/sshd_config");
	foreach my $ip (@jailips) {
	    mysystem("echo 'ListenAddress $ip' >> ".
		     "$vdir/etc/ssh/sshd_config");
	}
    }
    else {
	mysystem("echo 'Port $sshdport' >> $vdir/etc/ssh/sshd_config");
    }

    tmcccopycache($vnodeid, "$vdir");
    if (-e TMTOPOMAP()) {
	mysystem("cp -fp " . TMTOPOMAP() . " $vdir/var/emulab/boot");
    }
    if (-e TMLTMAP()) {
	mysystem("cp -fp " . TMLTMAP() . " $vdir/var/emulab/boot");
    }
    if (-e TMLTPMAP()) {
	mysystem("cp -fp " . TMLTPMAP() . " $vdir/var/emulab/boot");
    }

    #
    # Stash the control net IP if not the same as the host IP
    #
    if ($IP ne $hostip) {
	mysystem("echo $IP > $vdir/var/emulab/boot/myip");
    }

    #
    # This is picked up by rc.invserver and put into the environment (tmcc).
    #
    mysystem("echo $vnodeid > $vdir/var/emulab/boot/realname");

    #
    # Password/group file stuff. If remote the jail picks them up. Locally
    # we start with current set of accounts on the physnode, since this is
    # more efficient (less tmcd work).
    #
    if (REMOTE()) {
	mysystem("cp -p $ETCDIR/group $ETCDIR/passwd $vdir/etc");
	mysystem("cp -p $ETCDIR/gshadow $ETCDIR/shadow $vdir/etc");
    }
    else {
	# Already copied /etc above, but also need the .db files so that
	# inner rc.accounts knows.
	mysystem("cp -p $DBDIR/passdb $DBDIR/groupdb $vdir/$DBDIR");
    }

    #
    # Mount the usual directories inside the jail. Use linux loopback
    # mounts which will remount an NFS filesystem.
    #
    if (! REMOTE()) {
	mysystem("$BINDIR/rc/rc.mounts -j $vnodeid $vdir 0 boot");
    }
    TBDebugTimeStamp("mkvserver: mounting NFS filesystems done");

    # Remove this file; rc.invserver will create it at the very end.
    # which indicates the inner environment is running reasonably well.
    unlink("$vdir/$BOOTDIR/vrunning")
	if (-e "$vdir/$BOOTDIR/vrunning");

    #
    # vserver 2.2 does not support loopback devices inside, so redirect
    # localhost so that testbed daemons can find the local pubsubd. This
    # will come out when we have the virtual network stuff.
    #
    mysystem("echo '$IP\t\tlocalhost loghost' > $vdir/etc/hosts");
}

#
# Reuse existing vserver.
#
sub upvserver($)<commit_msg>Get rid of the localhost hack; the new image is 2.3 vserver and it does the loopback device properly inside vservers. Also remove the tmcd pem files from inside the vserver so that forced to go through the proxy.<commit_after>#!/usr/bin/perl -w
#
# EMULAB-COPYRIGHT
# Copyright (c) 2008 University of Utah and the Flux Group.
# All rights reserved.
#
# Kernel, jail, netstat, route, ifconfig, ipfw, header files.
#
use strict;
use English;
use Getopt::Std;
use Fcntl;
use IO::Handle;
use Socket;
use Fcntl ':flock';

# Drag in path stuff so we can find emulab stuff. Also untaints path.
BEGIN { require "/etc/emulab/paths.pm"; import emulabpaths; }

use libsetup qw(REMOTE LOCALROOTFS TMTOPOMAP TMLTMAP TMLTPMAP
		TBDebugTimeStamp);
use libtmcc;

#
# Questions:
#

#
# Create a jailed environment. There are some stub files stored in
# /etc/jail that copied into the jail.
#
sub usage()
{
    print("Usage: mkvserver.pl [-V] [-s] [-i <ipaddr>] [-p <pid>] ".
	  "[-h <hostname>] <vnodeid>\n");
    exit(-1);
}
my  $optlist = "Vi:p:e:sh:";

#
# Only real root can run this script.
#
if ($UID) {
    die("Must be root to run this script!\n");
}

#
# Catch ^C and exit with error. 
#
my $leaveme = 0;
sub handler ($) {
    my ($signame) = @_;
    
    $SIG{INT}  = 'IGNORE';
    $SIG{USR1} = 'IGNORE';
    $SIG{TERM} = 'IGNORE';
    $SIG{HUP}  = 'IGNORE';

    if ($signame eq 'USR1') {
	$leaveme = 1;
    }
    fatal("Caught a SIG${signame}! Killing the vserver ...");
}
$SIG{INT}  = \&handler;
$SIG{USR1} = \&handler;
$SIG{HUP}  = \&handler;
$SIG{TERM} = 'IGNORE';

#
# Turn off line buffering on output
#
STDOUT->autoflush(1);
STDERR->autoflush(1);

# XXX
my $JAILCNET	   = "172.16.0.0";
my $JAILCNETMASK   = "255.240.0.0";

#
# Locals
#
my $JAILPATH	   = "/var/emulab/jails";
my $ETCVSERVER     = "/usr/local/etc/emulab/vserver";
my $VSERVER	   = "/usr/sbin/vserver";
my $TMCC	   = "$BINDIR/tmcc";
my $VSERVERDIR     = "/vservers";
my $JAILCONFIG     = "jailconfig";
my @ROOTCPDIRS     = ("etc", "root");
my @ROOTMKDIRS     = ("dev", "tmp", "var", "usr", "proc", "users", "lib",
		      "bin", "sbin", "home");
my @ROOTMNTDIRS    = ("bin", "sbin", "usr", "lib");
my @EMUVARDIRS	   = ("logs", "db", "jails", "boot", "lock");
my $IP;
my $IPMASK;
my $PID;
my $VDIR;
my $CDIR;
my $idnumber;
my $jailhostname;
my $jailpid;
my $tmccpid;
my $debug	   = 1;
my @mntpoints      = ();
my $USEVCNETROUTES = 0;
my @controlroutes  = ();
my $interactive    = 0;
my $cleaning	   = 0;

# This stuff is passed from tmcd, which we parse into a config string
# and an option set.
my %jailconfig  = ();
my $sshdport    = 50000;	# Bogus default, good for testing.
my $jailflags   = 0;
my @jailips     = ();		# List of jail IPs (for routing table).

#
# Protos
#
sub mkvserver($);
sub upvserver($);
sub LoopMount($$);
sub PreparePhysNode();
sub fatal($);
sub getjailconfig($);
sub setjailoptions();
sub startproxy($);
sub mysystem($);
sub cleanup();

#
# Parse command arguments. Once we return from getopts, all that should be
# left are the required arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
    usage();
}
if (@ARGV != 1) {
    usage();
}
my $vnodeid = $ARGV[0];

#
# Untaint the arguments.
#
if ($vnodeid =~ /^([-\w\/\.]+)$/) {
    $vnodeid = $1;
    $jailhostname = $1;
}
else {
    die("Tainted argument $vnodeid!\n");
}

if (defined($options{'s'})) {
    $interactive = 1;
}

TBDebugTimeStamp("mkjail starting to do real work");

#
# Get the parent IP.
# 
my $hostname = `hostname`;
my $hostip;

# Untaint and strip newline.
if ($hostname =~ /^([-\w\.]+)$/) {
    $hostname = $1;

    my (undef,undef,undef,undef,@ipaddrs) = gethostbyname($hostname);
    $hostip = inet_ntoa($ipaddrs[0]);
}

#
# If no IP, then it defaults to our hostname's IP, *if* none is provided
# by the jail configuration file. That check is later. 
# 
if (defined($options{'i'})) {
    $IP = $options{'i'};

    if ($IP =~ /^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/) {
	$IP = $1;
    }
    else {
	die("Tainted argument $IP!\n");
    }
}

if (defined($options{'p'})) {
    $PID = $options{'p'};

    if ($PID =~ /^([-\@\w]+)$/) {
	$PID = $1;
    }
    else {
	die("Tainted argument $PID.");
    }
}

if (defined($options{'h'})) {
    $jailhostname = $options{'h'};

    if ($jailhostname =~ /^([-\w\.]+)$/) {
	$jailhostname = $1;
    }
    else {
	die("Tainted argument $jailhostname.");
    }
}

#
# In most cases, the $vnodeid directory will have been created by the caller,
# and a config file possibly dropped in.
# When debugging, we have to create it here. 
# 
chdir($JAILPATH) or
    die("Could not chdir to $JAILPATH: $!\n");

if (! -e $vnodeid) {
    mkdir($vnodeid, 0770) or
	fatal("Could not mkdir $vnodeid in $JAILPATH: $!");
}
else {
    TBDebugTimeStamp("mkjail getting jail config");
    getjailconfig("$JAILPATH/$vnodeid");
}

my $phys_cnet_if = `control_interface`;
chomp($phys_cnet_if);

#
# See if special options supported, and if so setup args as directed.
#
TBDebugTimeStamp("mkjail setting jail options");
setjailoptions();

# Do some prep stuff on the physical node if this is the first vserver.
PreparePhysNode();

print("Setting up jail for $vnodeid using $IP\n")
    if ($debug);

$VDIR = "$VSERVERDIR/$vnodeid";
$CDIR = "/etc/vservers/$vnodeid";

#
# Create the vserver.
#
if (-e $VDIR) {
    #
    # Try to pick up where we left off.
    #
    TBDebugTimeStamp("mkjail restoring root fs");
    upvserver("$vnodeid");
}
else {
    #
    # Create the root filesystem.
    #
    TBDebugTimeStamp("Creating vserver");
    mkvserver("$vnodeid");
}
TBDebugTimeStamp("mkjail done with root fs");
startproxy("$vnodeid");

#
# Start the vserver. If all goes well, this will exit cleanly, with the
# vserver running in its new context. Still, lets protect it with a timer
# since it might get hung up inside and we do not want to get stuck here.
#
my $childpid = fork();
if ($childpid) {
    #local $SIG{ALRM} = sub { kill("TERM", $childpid); };
    #alarm 30;
    waitpid($childpid, 0);
    #alarm 0;

    #
    # If failure then cleanup.
    #
    if ($?) {
	fatal("mkvserver: $vnodeid vserver startup exited with $?");
    }
}
else {
    $SIG{TERM} = 'DEFAULT';

    TBDebugTimeStamp("mkvserver: starting the vserver");

    exec("$VSERVER $vnodeid start");
    die("*** $0:\n".
	"    exec failed to start the vserver!\n");
}

#
# If this file does not exist, the inner setup failed somehow. Stop now.
#
fatal("vserver did not appear to set up properly. Exiting ...")
    if (! -e "$VDIR/$BOOTDIR/vrunning");

print "vserver for $vnodeid started. Waiting ...\n";
$jailpid = fork();
if ($jailpid) {
    #
    # We do not really care about the exit status of the jail, we just want
    # to know when it dies inside.
    #
    while (1) {
	my $kidpid = waitpid(-1, 0);

	if ($kidpid == $jailpid) {
	    undef($jailpid);
	    last;
	}
	if ($kidpid == $tmccpid) {
	    print("TMCC proxy exited with status $?. Restarting ...\n");
	    startproxy("$vnodeid");
	    next;
	}
	print("Unknown child $kidpid exited with status $?!\n");
    }
}
else {
    $SIG{TERM} = 'DEFAULT';

    exec("$VSERVER $vnodeid exec sleep 1000000");
    die("*** $0:\n".
	"    exec failed to start the jail!\n");
}
print "vserver for $vnodeid has died. Cleaning up ...\n";
cleanup();
exit(0);

#
# Create a root filesystem for the vserver.
#
sub mkvserver($)
{
    my ($vnodeid) = @_;
    my $interface;

    if (defined($IP)) {
	system("/usr/lib/util-vserver/mask2prefix $JAILCNETMASK");
	my $prefix = $? >> 8;
	$interface = "${vnodeid}=${phys_cnet_if}:${IP}/${prefix}";
    }
    else {
	$interface = "nodev:0.0.0.0/0";
    }

    # Create the skeleton vserver. It will be mostly empty.
    mysystem("$VSERVER $vnodeid build --force -m skeleton ".
	     "--hostname $jailhostname --interface $interface ".
	     "--flags persistent");

    # The filesystem for the vserver lands here.
    my $vdir = $VDIR;
    # The configuration directory is here.
    my $cdir = $CDIR;

    #
    # Copy in the top level directories. 
    #
    foreach my $dir (@ROOTCPDIRS) {
	mysystem("rsync -a /$dir $vdir");
    }
    TBDebugTimeStamp("mkvserver: Copying root cp dirs done!");

    #
    # Set vserver "capabilities".
    #
    # Allows binding to TCP/UDP sockets below 1024
    mysystem("echo 'NET_BIND_SERVICE' > $cdir/bcapabilities");

    #
    # Clean out some stuff from /etc.
    #
    mysystem("/bin/rm -rf $vdir/etc/rc.d/rc*.d/*");

    #
    # Make some other directories that are need in /root.
    #
    foreach my $dir (@ROOTMKDIRS) {
	if (! -e "$vdir/$dir") {
	    mkdir("$vdir/$dir", 0755) or
		fatal("Could not mkdir '$dir' in $vdir: $!");
	}
    }
    TBDebugTimeStamp("mkvserver: Creating root mkdir dirs done!");

    #
    # Mount (read-only) these other directories to save space.
    #
    foreach my $dir (@ROOTMNTDIRS) {
	LoopMount("/$dir", "$vdir/$dir");
    }
    TBDebugTimeStamp("mkvserver: Mounting root dirs done!");

    # /tmp is special.
    mysystem("chmod 1777 $vdir/tmp");

    #
    # Duplicate the /var hierarchy without the contents.
    #
    open(VARDIRS, "find /var -type d -print |")
	or fatal("Could not start find on /var");
    while (<VARDIRS>) {
	my $dir = $_;
	chomp($dir);

	mysystem("rsync -dlptgoD $dir $vdir$dir");
    }
    close(VARDIRS);

    #
    # Get a list of all the plain files and create zero length versions
    # in the new var.
    #
    opendir(DIR, "/var/log") or
	fatal("Cannot opendir /var/log: $!");
    my @logs = grep { -f "/var/log/$_" } readdir(DIR);
    closedir(DIR);

    foreach my $log (@logs) {
	mysystem("touch $vdir/var/log/$log");
    }
    TBDebugTimeStamp("mkvserver: finished setting up /var");
    
    # Create a local logs directory. 
    mysystem("mkdir -p $vdir/local/logs")
	if (! -e "$vdir/local/logs");

    #
    # Some security stuff; remove files that would enable it to talk to
    # tmcd directly (only imortant on remote nodes). Must go through proxy.
    #
    mysystem("rm -f $vdir/$ETCDIR/*.pem");
    
    #
    # Now a bunch of stuff to set up a nice environment in the jail.
    #
    mysystem("ln -s ../../init.d/syslog $vdir/etc/rc3.d/S80syslog");
    mysystem("ln -s ../../init.d/sshd $vdir/etc/rc3.d/S85sshd");
    mysystem("ln -s ../../init.d/syslog $vdir/etc/rc6.d/K85syslog");
    mysystem("ln -s ../../init.d/sshd $vdir/etc/rc6.d/K80sshd");
    mysystem("cp -p $ETCVSERVER/rc.invserver $vdir/etc/rc3.d/S99invserver");
    mysystem("cp -p $ETCVSERVER/rc.invserver $vdir/etc/rc6.d/K99invserver");

    # Kill anything that uses /dev/console in syslog; will not work.
    mysystem("sed -i.bak ".
	     "-e '/console/d' ".
	     "-e '/users/d' ".
	     "$vdir/etc/syslog.conf");
    
    # Premunge the sshd config: no X11 forwarding and get rid of old
    # ListenAddresses.
    #
    mysystem("sed -i.bak ".
	     "-e 's/^X11Forwarding.*yes/X11Forwarding no/' ".
	     "-e '/^ListenAddress/d' ".
	     "$vdir/etc/ssh/sshd_config");
    
    # Port/Address for sshd.
    if ($IP ne $hostip) {
	# XXX This can come out (sshd bind to port 22) once we have
	# the network stack stuff in the kernel.
	mysystem("echo 'Port $sshdport' >> $vdir/etc/ssh/sshd_config");

	mysystem("echo 'ListenAddress $IP' >> ".
		 "$vdir/etc/ssh/sshd_config");
	foreach my $ip (@jailips) {
	    mysystem("echo 'ListenAddress $ip' >> ".
		     "$vdir/etc/ssh/sshd_config");
	}
    }
    else {
	mysystem("echo 'Port $sshdport' >> $vdir/etc/ssh/sshd_config");
    }

    tmcccopycache($vnodeid, "$vdir");
    if (-e TMTOPOMAP()) {
	mysystem("cp -fp " . TMTOPOMAP() . " $vdir/var/emulab/boot");
    }
    if (-e TMLTMAP()) {
	mysystem("cp -fp " . TMLTMAP() . " $vdir/var/emulab/boot");
    }
    if (-e TMLTPMAP()) {
	mysystem("cp -fp " . TMLTPMAP() . " $vdir/var/emulab/boot");
    }

    #
    # Stash the control net IP if not the same as the host IP
    #
    if ($IP ne $hostip) {
	mysystem("echo $IP > $vdir/var/emulab/boot/myip");
    }

    #
    # This is picked up by rc.invserver and put into the environment (tmcc).
    #
    mysystem("echo $vnodeid > $vdir/var/emulab/boot/realname");

    #
    # Password/group file stuff. If remote the jail picks them up. Locally
    # we start with current set of accounts on the physnode, since this is
    # more efficient (less tmcd work).
    #
    if (REMOTE()) {
	mysystem("cp -p $ETCDIR/group $ETCDIR/passwd $vdir/etc");
	mysystem("cp -p $ETCDIR/gshadow $ETCDIR/shadow $vdir/etc");
    }
    else {
	# Already copied /etc above, but also need the .db files so that
	# inner rc.accounts knows.
	mysystem("cp -p $DBDIR/passdb $DBDIR/groupdb $vdir/$DBDIR");
    }

    #
    # Mount the usual directories inside the jail. Use linux loopback
    # mounts which will remount an NFS filesystem.
    #
    if (! REMOTE()) {
	mysystem("$BINDIR/rc/rc.mounts -j $vnodeid $vdir 0 boot");
    }
    TBDebugTimeStamp("mkvserver: mounting NFS filesystems done");

    # Remove this file; rc.invserver will create it at the very end.
    # which indicates the inner environment is running reasonably well.
    unlink("$vdir/$BOOTDIR/vrunning")
	if (-e "$vdir/$BOOTDIR/vrunning");

}

#
# Reuse existing vserver.
#
sub upvserver($)
{'fraction_non_alphanumeric': 0.16873468137254902, 'fraction_numerical': 0.009497549019607842, 'mean_word_length': 2.935644310474755, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '480324', 'n_tokens_mistral': 10193, 'n_tokens_neox': 9840, 'n_words': 2774}