<commit_before>#include <stdio.h>
#include <stdlib.h>
#include <signal.h>
#include <limits.h>
#include <unistd.h>
#include <errno.h>
#include <dirent.h>
#include <string.h>

#include <fcntl.h>
#include <grp.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <sys/vfs.h>
#include <sys/ptrace.h>
#include <sys/wait.h>
#include <sys/file.h>
#include <sys/shm.h>
#include <sys/mount.h>

#include <sched.h>

#include <sys/sendfile.h>

#include "compiler.h"
#include "types.h"

#include "image.h"
#include "util.h"
#include "log.h"
#include "syscall.h"
#include "restorer.h"
#include "sockets.h"
#include "sk-packet.h"
#include "lock.h"
#include "files.h"
#include "files-reg.h"
#include "pipes.h"
#include "fifo.h"
#include "sk-inet.h"
#include "eventfd.h"
#include "eventpoll.h"
#include "signalfd.h"
#include "proc_parse.h"
#include "restorer-blob.h"
#include "crtools.h"
#include "namespaces.h"
#include "shmem.h"
#include "mount.h"
#include "inotify.h"
#include "pstree.h"
#include "net.h"
#include "tty.h"

#include "protobuf.h"
#include "protobuf/sa.pb-c.h"
#include "protobuf/itimer.pb-c.h"
#include "protobuf/vma.pb-c.h"

static struct pstree_item *current;

static int restore_task_with_children(void *);
static int sigreturn_restore(pid_t pid, CoreEntry *core);
static int prepare_restorer_blob(void);

static LIST_HEAD(rst_vma_list);
static int rst_nr_vmas;
static void *premmapped_addr;
static unsigned long premmapped_len;

static int shmem_remap(void *old_addr, void *new_addr, unsigned long size)
{
	void *ret;

	ret = mremap(old_addr, size, size,
			MREMAP_FIXED | MREMAP_MAYMOVE, new_addr);
	if (new_addr != ret) {
		pr_perror("mremap failed");
		return -1;
	}

	return 0;
}

static int crtools_prepare_shared(void)
{
	if (prepare_shared_fdinfo())
		return -1;

	/* Connections are unlocked from crtools */
	if (collect_inet_sockets())
		return -1;

	if (tty_prep_fds())
		return -1;

	return 0;
}

static int root_prepare_shared(void)
{
	int ret = 0;
	struct pstree_item *pi;

	pr_info("Preparing info about shared resources\n");

	if (prepare_shmem_restore())
		return -1;

	if (prepare_shared_tty())
		return -1;

	if (prepare_shared_reg_files())
		return -1;

	if (collect_reg_files())
		return -1;

	if (collect_pipes())
		return -1;

	if (collect_fifo())
		return -1;

	if (collect_unix_sockets())
		return -1;

	if (collect_packet_sockets())
		return -1;

	if (collect_eventfd())
		return -1;

	if (collect_eventpoll())
		return -1;

	if (collect_signalfd())
		return -1;

	if (collect_inotify())
		return -1;

	if (collect_tty())
		return -1;

	for_each_pstree_item(pi) {
		if (pi->state == TASK_HELPER)
			continue;

		ret = prepare_shmem_pid(pi->pid.virt);
		if (ret < 0)
			break;

		ret = prepare_fd_pid(pi->pid.virt, pi->rst);
		if (ret < 0)
			break;
	}

	if (ret < 0)
		goto err;

	mark_pipe_master();

	ret = tty_setup_slavery();
	if (ret)
		goto err;

	ret = resolve_unix_peers();
	if (ret)
		goto err;

	ret = prepare_restorer_blob();
	if (ret)
		goto err;

	show_saved_shmems();
	show_saved_files();
err:
	return ret;
}

/* Map a private vma, if it is not mapped by a parrent yet */
static int map_private_vma(pid_t pid, struct vma_area *vma, void *tgt_addr,
			struct vma_area **pvma, struct list_head *pvma_list)
{
	int ret;
	void *addr, *paddr = NULL;
	unsigned long nr_pages;
	struct vma_area *p = *pvma;

	if (vma_entry_is(&vma->vma, VMA_FILE_PRIVATE)) {
		ret = get_filemap_fd(pid, &vma->vma);
		if (ret < 0) {
			pr_err("Can't fixup fd\n");
			return -1;
		}
		vma->vma.fd = ret;
		/* shmid will be used for a temporary address */
		vma->vma.shmid = 0;
	}

	nr_pages = vma_entry_len(&vma->vma) / PAGE_SIZE;
	vma->page_bitmap = xzalloc(BITS_TO_LONGS(nr_pages) * sizeof(long));
	if (vma->page_bitmap == NULL)
		return -1;

	list_for_each_entry_continue(p, pvma_list, list) {
		if (p->vma.start > vma->vma.start)
			 break;

		if (p->vma.end == vma->vma.end &&
		    p->vma.start == vma->vma.start) {
			pr_info("COW 0x%016lx-0x%016lx 0x%016lx vma\n",
				vma->vma.start, vma->vma.end, vma->vma.pgoff);
			paddr = (void *) vma_premmaped_start(&p->vma);
			break;
		}

	}

	*pvma = p;

	if (paddr == NULL) {
		pr_info("Map 0x%016lx-0x%016lx 0x%016lx vma\n",
			vma->vma.start, vma->vma.end, vma->vma.pgoff);

		addr = mmap(tgt_addr, vma_entry_len(&vma->vma),
				vma->vma.prot | PROT_WRITE,
				vma->vma.flags | MAP_FIXED,
				vma->vma.fd, vma->vma.pgoff);

		if (addr == MAP_FAILED) {
			pr_perror("Unable to map ANON_VMA");
			return -1;
		}
	} else {
		vma->ppage_bitmap = p->page_bitmap;

		addr = mremap(paddr, vma_area_len(vma), vma_area_len(vma),
				MREMAP_FIXED | MREMAP_MAYMOVE, tgt_addr);
		if (addr != tgt_addr) {
			pr_perror("Unable to remap a private vma");
			return -1;
		}

	}

	vma_premmaped_start(&(vma->vma)) = (unsigned long) addr;

	if (vma_entry_is(&vma->vma, VMA_FILE_PRIVATE))
		close(vma->vma.fd);

	return 0;
}

static int restore_priv_vma_content(pid_t pid)
{
	struct vma_area *vma;
	int fd, ret = 0;

	unsigned int nr_restored = 0;
	unsigned int nr_shared = 0;
	unsigned int nr_droped = 0;

	vma = list_first_entry(&rst_vma_list, struct vma_area, list);

	fd = open_image_ro(CR_FD_PAGES, pid);
	if (fd < 0)
		return -1;

	/*
	 * Read page contents.
	 */
	while (1) {
		u64 va, page_offset;
		char buf[PAGE_SIZE];
		void *p;

		ret = read(fd, &va, sizeof(va));
		if (!ret)
			break;

		if (ret != sizeof(va)) {
			pr_err("Bad mapping page size %d\n", ret);
			return -1;
		}

		BUG_ON(va < vma->vma.start);

		while (va >= vma->vma.end) {
			BUG_ON(vma->list.next == &rst_vma_list);
			vma = list_entry(vma->list.next, struct vma_area, list);
		}

		page_offset = (va - vma->vma.start) / PAGE_SIZE;

		set_bit(page_offset, vma->page_bitmap);
		if (vma->ppage_bitmap)
			clear_bit(page_offset, vma->ppage_bitmap);

		ret = read(fd, buf, PAGE_SIZE);
		if (ret != PAGE_SIZE) {
			pr_err("Can'r read mapping page %d\n", ret);
			return -1;
		}

		p = (void *) (va - vma->vma.start +
					vma_premmaped_start(&vma->vma));
		if (memcmp(p, buf, PAGE_SIZE) == 0) {
			nr_shared++;
			continue;
		}

		memcpy(p, buf, PAGE_SIZE);
		nr_restored++;
	}
	close(fd);

	/* Remove pages, which were not shared with a child */
	list_for_each_entry(vma, &rst_vma_list, list) {
		unsigned long size, i = 0;
		void *addr = (void *) vma_premmaped_start(&vma->vma);

		if (vma->ppage_bitmap == NULL)
			continue;

		size = vma_entry_len(&vma->vma) / PAGE_SIZE;
		while (1) {
			/* Find all pages, which are not shared with this child */
			i = find_next_bit(vma->ppage_bitmap, size, i);

			if ( i >= size)
				break;

			ret = madvise(addr + PAGE_SIZE * i,
						PAGE_SIZE, MADV_DONTNEED);
			if (ret < 0) {
				pr_perror("madvise failed\n");
				return -1;
			}
			i++;
			nr_droped++;
		}
	}

	pr_info("nr_restored_pages: %d\n", nr_restored);
	pr_info("nr_shared_pages:   %d\n", nr_shared);
	pr_info("nr_droped_pages:   %d\n", nr_droped);

	return 0;
}

static int read_vmas(int pid)
{
	int fd, ret = 0;
	LIST_HEAD(old);
	struct vma_area *pvma, *vma;
	unsigned long priv_size = 0;
	void *addr;

	void *old_premmapped_addr = NULL;
	unsigned long old_premmapped_len, pstart = 0;

	rst_nr_vmas = 0;
	list_replace_init(&rst_vma_list, &old);

	/* Skip errors, because a zombie doesn't have an image of vmas */
	fd = open_image_ro(CR_FD_VMAS, pid);
	if (fd < 0) {
		if (errno != ENOENT)
			ret = fd;
		goto out;
	}

	while (1) {
		struct vma_area *vma;
		VmaEntry *e;

		ret = -1;
		vma = alloc_vma_area();
		if (!vma)
			break;

		ret = pb_read_one_eof(fd, &e, PB_VMAS);
		if (ret <= 0)
			break;

		rst_nr_vmas++;
		list_add_tail(&vma->list, &rst_vma_list);

		if (e->fd != -1) {
			ret = -1;
			pr_err("Error in vma->fd setting (%Ld)\n",
					(unsigned long long)e->fd);
			break;
		}

		vma->vma = *e;
		vma_entry__free_unpacked(e, NULL);

		if (!vma_priv(&vma->vma))
			continue;

		priv_size += vma_area_len(vma);
	}

	/* Reserve a place for mapping private vma-s one by one */
	addr = mmap(NULL, priv_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
	if (addr == MAP_FAILED) {
		pr_perror("Unable to reserve memory");
		return -1;
	}

	old_premmapped_addr = premmapped_addr;
	old_premmapped_len = premmapped_len;
	premmapped_addr = addr;
	premmapped_len = priv_size;

	pvma = list_entry(&old, struct vma_area, list);

	list_for_each_entry(vma, &rst_vma_list, list) {
		if (pstart > vma->vma.start) {
			ret = -1;
			pr_err("VMA-s are not sorted in the image file\n");
			break;
		}
		pstart = vma->vma.start;

		if (!vma_priv(&vma->vma))
			continue;

		ret = map_private_vma(pid, vma, addr, &pvma, &old);
		if (ret < 0)
			break;

		addr += vma_area_len(vma);
	}

	if (ret == 0)
		ret = restore_priv_vma_content(pid);
	close(fd);

out:
	while (!list_empty(&old)) {
		vma = list_first_entry(&old, struct vma_area, list);
		list_del(&vma->list);
		xfree(vma);
	}

	if (old_premmapped_addr &&
	    munmap(old_premmapped_addr, old_premmapped_len)) {
		pr_perror("Unable to unmap %p(%lx)",
				old_premmapped_addr, old_premmapped_len);
		return -1;
	}


	return ret;
}

static int open_vmas(int pid, struct list_head *vmas)
{
	struct vma_area *vma;
	int ret = 0;

	list_for_each_entry(vma, vmas, list) {
		if (!(vma_entry_is(&vma->vma, VMA_AREA_REGULAR)))
			continue;

		pr_info("Opening 0x%016lx-0x%016lx 0x%016lx (%x) vma\n",
				vma->vma.start, vma->vma.end,
				vma->vma.pgoff, vma->vma.status);

		if (vma_entry_is(&vma->vma, VMA_AREA_SYSVIPC))
			ret = vma->vma.shmid;
		else if (vma_entry_is(&vma->vma, VMA_ANON_SHARED))
			ret = get_shmem_fd(pid, &vma->vma);
		else if (vma_entry_is(&vma->vma, VMA_FILE_SHARED))
			ret = get_filemap_fd(pid, &vma->vma);
		else if (vma_entry_is(&vma->vma, VMA_AREA_SOCKET))
			ret = get_socket_fd(pid, &vma->vma);
		else
			continue;

		if (ret < 0) {
			pr_err("Can't fixup fd\n");
			break;
		}

		pr_info("\t`- setting %d as mapping fd\n", ret);
		vma->vma.fd = ret;
	}

	return ret < 0 ? -1 : 0;
}

static int prepare_and_sigreturn(int pid, CoreEntry *core)
{
	int err;

	err = open_vmas(pid, &rst_vma_list);
	if (err)
		return err;

	return sigreturn_restore(pid, core);
}

static rt_sigaction_t sigchld_act;
static int prepare_sigactions(int pid)
{
	rt_sigaction_t act, oact;
	int fd_sigact;
	SaEntry *e;
	int sig;
	int ret = -1;

	fd_sigact = open_image_ro(CR_FD_SIGACT, pid);
	if (fd_sigact < 0)
		return -1;

	for (sig = 1; sig < SIGMAX; sig++) {
		if (sig == SIGKILL || sig == SIGSTOP)
			continue;

		ret = pb_read_one(fd_sigact, &e, PB_SIGACT);
		if (ret < 0)
			break;

		ASSIGN_TYPED(act.rt_sa_handler, e->sigaction);
		ASSIGN_TYPED(act.rt_sa_flags, e->flags);
		ASSIGN_TYPED(act.rt_sa_restorer, e->restorer);
		ASSIGN_TYPED(act.rt_sa_mask.sig[0], e->mask);

		sa_entry__free_unpacked(e, NULL);

		if (sig == SIGCHLD) {
			sigchld_act = act;
			continue;
		}
		/*
		 * A pure syscall is used, because glibc
		 * sigaction overwrites se_restorer.
		 */
		ret = sys_sigaction(sig, &act, &oact, sizeof(rt_sigset_t));
		if (ret == -1) {
			pr_err("%d: Can't restore sigaction: %m\n", pid);
			goto err;
		}
	}

err:
	close_safe(&fd_sigact);
	return ret;
}

static int pstree_wait_helpers()
{
	struct pstree_item *pi;

	list_for_each_entry(pi, &current->children, sibling) {
		int status, ret;

		if (pi->state != TASK_HELPER)
			continue;

		/* Check, that a helper completed. */
		ret = waitpid(pi->pid.virt, &status, 0);
		if (ret == -1) {
			if (errno == ECHILD)
				continue; /* It has been waited in sigchld_handler */
			pr_err("waitpid(%d) failed\n", pi->pid.virt);
			return -1;
		}
		if (!WIFEXITED(status) || WEXITSTATUS(status)) {
			pr_err("%d exited with non-zero code (%d,%d)\n", pi->pid.virt,
				WEXITSTATUS(status), WTERMSIG(status));
			return -1;
		}

	}

	return 0;
}


static int restore_one_alive_task(int pid, CoreEntry *core)
{
	pr_info("Restoring resources\n");

	if (pstree_wait_helpers())
		return -1;

	if (prepare_fds(current))
		return -1;

	if (prepare_fs(pid))
		return -1;

	if (prepare_sigactions(pid))
		return -1;

	log_closedir();

	return prepare_and_sigreturn(pid, core);
}

static void zombie_prepare_signals(void)
{
	sigset_t blockmask;
	int sig;
	struct sigaction act;

	sigfillset(&blockmask);
	sigprocmask(SIG_UNBLOCK, &blockmask, NULL);

	memset(&act, 0, sizeof(act));
	act.sa_handler = SIG_DFL;

	for (sig = 1; sig < SIGMAX; sig++)
		sigaction(sig, &act, NULL);
}

#define SIG_FATAL_MASK	(	\
		(1 << SIGHUP)	|\
		(1 << SIGINT)	|\
		(1 << SIGQUIT)	|\
		(1 << SIGILL)	|\
		(1 << SIGTRAP)	|\
		(1 << SIGABRT)	|\
		(1 << SIGIOT)	|\
		(1 << SIGBUS)	|\
		(1 << SIGFPE)	|\
		(1 << SIGKILL)	|\
		(1 << SIGUSR1)	|\
		(1 << SIGSEGV)	|\
		(1 << SIGUSR2)	|\
		(1 << SIGPIPE)	|\
		(1 << SIGALRM)	|\
		(1 << SIGTERM)	|\
		(1 << SIGXCPU)	|\
		(1 << SIGXFSZ)	|\
		(1 << SIGVTALRM)|\
		(1 << SIGPROF)	|\
		(1 << SIGPOLL)	|\
		(1 << SIGIO)	|\
		(1 << SIGSYS)	|\
		(1 << SIGUNUSED)|\
		(1 << SIGSTKFLT)|\
		(1 << SIGPWR)	 \
	)

static inline int sig_fatal(int sig)
{
	return (sig > 0) && (sig < SIGMAX) && (SIG_FATAL_MASK & (1 << sig));
}

struct task_entries *task_entries;

static int restore_one_fake(int pid)
{
	/* We should wait here, otherwise last_pid will be changed. */
	futex_wait_while(&task_entries->start, CR_STATE_FORKING);
	futex_wait_while(&task_entries->start, CR_STATE_RESTORE_PGID);
	return 0;
}

static int restore_one_zombie(int pid, int exit_code)
{
	pr_info("Restoring zombie with %d code\n", exit_code);

	if (task_entries != NULL) {
		restore_finish_stage(CR_STATE_RESTORE);
		zombie_prepare_signals();
		restore_finish_stage(CR_STATE_RESTORE_SIGCHLD);
	}

	if (exit_code & 0x7f) {
		int signr;

		signr = exit_code & 0x7F;
		if (!sig_fatal(signr)) {
			pr_warn("Exit with non fatal signal ignored\n");
			signr = SIGABRT;
		}

		if (kill(pid, signr) < 0)
			pr_perror("Can't kill myself, will just exit");

		exit_code = 0;
	}

	exit((exit_code >> 8) & 0x7f);

	/* never reached */
	BUG_ON(1);
	return -1;
}

static int check_core(CoreEntry *core)
{
	int ret = -1;

	if (core->mtype != CORE_ENTRY__MARCH__X86_64) {
		pr_err("Core march mismatch %d\n", (int)core->mtype);
		goto out;
	}

	if (!core->tc) {
		pr_err("Core task state data missed\n");
		goto out;
	}

	if (core->tc->task_state != TASK_DEAD) {
		if (!core->ids) {
			pr_err("Core IDS data missed for non-zombie\n");
			goto out;
		}

		if (!core->thread_info) {
			pr_err("Core info data missed for non-zombie\n");
			goto out;
		}
	}

	ret = 0;
out:
	return ret < 0 ? ret : 0;
}

static int restore_one_task(int pid)
{
	int fd, ret;
	CoreEntry *core;

	if (current->state == TASK_HELPER)
		return restore_one_fake(pid);

	fd = open_image_ro(CR_FD_CORE, pid);
	if (fd < 0)
		return -1;

	ret = pb_read_one(fd, &core, PB_CORE);
	close(fd);

	if (ret < 0)
		return -1;

	if (check_core(core)) {
		ret = -1;
		goto out;
	}

	switch ((int)core->tc->task_state) {
	case TASK_ALIVE:
		ret = restore_one_alive_task(pid, core);
		break;
	case TASK_DEAD:
		ret = restore_one_zombie(pid, core->tc->exit_code);
		break;
	default:
		pr_err("Unknown state in code %d\n", (int)core->tc->task_state);
		ret = -1;
		break;
	}

out:
	core_entry__free_unpacked(core, NULL);
	return ret;
}

/* All arguments should be above stack, because it grows down */
struct cr_clone_arg {
	char stack[PAGE_SIZE];
	char stack_ptr[0];
	struct pstree_item *item;
	unsigned long clone_flags;
	int fd;
};

static void write_pidfile(char *pfname, int pid)
{
	int fd;

	fd = open(pfname, O_WRONLY | O_TRUNC | O_CREAT, 0600);
	if (fd == -1) {
		pr_perror("Can't open %s", pfname);
		kill(pid, SIGKILL);
		return;
	}

	dprintf(fd, "%d", pid);
	close(fd);
}

static inline int fork_with_pid(struct pstree_item *item, unsigned long ns_clone_flags)
{
	int ret = -1;
	struct cr_clone_arg ca;
	pid_t pid = item->pid.virt;

	pr_info("Forking task with %d pid (flags 0x%lx)\n", pid, ns_clone_flags);

	ca.item = item;
	ca.clone_flags = ns_clone_flags;

	if (!(ca.clone_flags & CLONE_NEWPID)) {
		char buf[32];

		ca.fd = open(LAST_PID_PATH, O_RDWR);
		if (ca.fd < 0) {
			pr_perror("%d: Can't open %s", pid, LAST_PID_PATH);
			goto err;
		}

		if (flock(ca.fd, LOCK_EX)) {
			close(ca.fd);
			pr_perror("%d: Can't lock %s", pid, LAST_PID_PATH);
			goto err;
		}

		snprintf(buf, sizeof(buf), "%d", pid - 1);
		if (write_img_buf(ca.fd, buf, strlen(buf)))
			goto err_unlock;
	} else {
		ca.fd = -1;
		BUG_ON(pid != INIT_PID);
	}

	if (ca.clone_flags & CLONE_NEWNET)
		/*
		 * When restoring a net namespace we need to communicate
		 * with the original (i.e. -- init) one. Thus, prepare for
		 * that before we leave the existing namespaces.
		 */
		if (netns_pre_create())
			goto err_unlock;

	ret = clone(restore_task_with_children, ca.stack_ptr,
			ca.clone_flags | SIGCHLD, &ca);

	if (ret < 0)
		pr_perror("Can't fork for %d", pid);

	if (ca.clone_flags & CLONE_NEWPID)
		item->pid.real = ret;

	if (opts.pidfile && root_item == item)
		write_pidfile(opts.pidfile, ret);

err_unlock:
	if (ca.fd >= 0) {
		if (flock(ca.fd, LOCK_UN))
			pr_perror("%d: Can't unlock %s", pid, LAST_PID_PATH);

		close(ca.fd);
	}
err:
	return ret;
}

static void sigchld_handler(int signal, siginfo_t *siginfo, void *data)
{
	struct pstree_item *pi;
	pid_t pid = siginfo->si_pid;
	int status;
	int exit;

	exit = siginfo->si_code & CLD_EXITED;
	status = siginfo->si_status;
	if (!current || status)
		goto err;

	/* Skip a helper if it was completed successfully */
	while (pid) {
		pid = waitpid(-1, &status, WNOHANG);
		if (pid <= 0)
			return;

		exit = WIFEXITED(status);
		status = exit ? WEXITSTATUS(status) : WTERMSIG(status);
		if (status)
			break;

		list_for_each_entry(pi, &current->children, sibling) {
			if (pi->state != TASK_HELPER)
				continue;
			if (pi->pid.virt == siginfo->si_pid)
				break;
		}

		if (&pi->sibling == &current->children)
			break; /* The process is not a helper */
	}

err:
	if (exit)
		pr_err("%d exited, status=%d\n", pid, status);
	else
		pr_err("%d killed by signal %d\n", pid, status);

	futex_abort_and_wake(&task_entries->nr_in_progress);
}

/*
 * FIXME Din't fail on xid restore failure. MySQL uses runaway
 * pgid and sid and there's nothing we can do about it yet :(
 */

static void xid_fail(void)
{
	exit(1);
}

static void restore_sid(void)
{
	pid_t sid;

	/*
	 * SID can only be reset to pid or inherited from parent.
	 * Thus we restore it right here to let our kids inherit
	 * one in case they need it.
	 *
	 * PGIDs are restored late when all tasks are forked and
	 * we can call setpgid() on custom values.
	 */

	if (current->pid.virt == current->sid) {
		pr_info("Restoring %d to %d sid\n", current->pid.virt, current->sid);
		sid = setsid();
		if (sid != current->sid) {
			pr_perror("Can't restore sid (%d)", sid);
			xid_fail();
		}
	} else {
		sid = getsid(getpid());
		if (sid != current->sid) {
			/* Skip the root task if it's not init */
			if (current == root_item && root_item->pid.virt != INIT_PID)
				return;
			pr_err("Requested sid %d doesn't match inherited %d\n",
					current->sid, sid);
			xid_fail();
		}
	}
}

static void restore_pgid(void)
{
	pid_t pgid;

	pr_info("Restoring %d to %d pgid\n", current->pid.virt, current->pgid);

	pgid = getpgrp();
	if (current->pgid == pgid)
		return;

	pr_info("\twill call setpgid, mine pgid is %d\n", pgid);
	if (setpgid(0, current->pgid) != 0) {
		pr_perror("Can't restore pgid (%d/%d->%d)", current->pid.virt, pgid, current->pgid);
		xid_fail();
	}
}

static int mount_proc(void)
{
	int ret;
	char proc_mountpoint[PATH_MAX];

	snprintf(proc_mountpoint, sizeof(proc_mountpoint), "crtools-proc.XXXXXX");
	if (mkdtemp(proc_mountpoint) == NULL) {
		pr_perror("mkdtemp failed %s", proc_mountpoint);
		return -1;
	}

	pr_info("Mount procfs in %s\n", proc_mountpoint);
	if (mount("proc", proc_mountpoint, "proc", MS_MGC_VAL, NULL)) {
		pr_perror("mount failed");
		ret = -1;
		goto out_rmdir;
	}

	ret = set_proc_mountpoint(proc_mountpoint);

	if (umount2(proc_mountpoint, MNT_DETACH) == -1) {
		pr_perror("Can't umount %s", proc_mountpoint);
		return -1;
	}

out_rmdir:
	if (rmdir(proc_mountpoint) == -1) {
		pr_perror("Can't remove %s", proc_mountpoint);
		return -1;
	}

	return ret;
}

static int restore_task_with_children(void *_arg)
{
	struct cr_clone_arg *ca = _arg;
	struct pstree_item *child;
	pid_t pid;
	int ret;
	sigset_t blockmask;

	close_safe(&ca->fd);

	current = ca->item;

	pid = getpid();
	if (current->pid.virt != pid) {
		pr_err("Pid %d do not match expected %d\n", pid, current->pid.virt);
		exit(-1);
	}

	ret = log_init_by_pid();
	if (ret < 0)
		exit(1);

	/* Restore root task */
	if (current->parent == NULL) {
		if (collect_mount_info())
			exit(-1);

		if (prepare_namespace(current->pid.virt, ca->clone_flags))
			exit(-1);

		/*
		 * We need non /proc proc mount for restoring pid and mount
		 * namespaces and do not care for the rest of the cases.
		 * Thus -- mount proc at custom location for any new namespace
		 */
		if (mount_proc())
			exit(-1);

		if (root_prepare_shared())
			exit(-1);
	}

	/*
	 * The block mask will be restored in sigresturn.
	 *
	 * TODO: This code should be removed, when a freezer will be added.
	 */
	sigfillset(&blockmask);
	sigdelset(&blockmask, SIGCHLD);
	ret = sigprocmask(SIG_BLOCK, &blockmask, NULL);
	if (ret) {
		pr_perror("%d: Can't block signals", current->pid.virt);
		exit(1);
	}

	if (read_vmas(pid))
		exit(1);

	pr_info("Restoring children:\n");
	list_for_each_entry(child, &current->children, sibling) {
		if (!restore_before_setsid(child))
			continue;

		BUG_ON(child->born_sid != -1 && getsid(getpid()) != child->born_sid);

		ret = fork_with_pid(child, 0);
		if (ret < 0)
			exit(1);
	}

	restore_sid();

	pr_info("Restoring children:\n");
	list_for_each_entry(child, &current->children, sibling) {
		if (restore_before_setsid(child))
			continue;
		ret = fork_with_pid(child, 0);
		if (ret < 0)
			exit(1);
	}

	if (current->pgid == current->pid.virt)
		restore_pgid();

	restore_finish_stage(CR_STATE_FORKING);

	if (current->pgid != current->pid.virt)
		restore_pgid();

	if (current->state != TASK_HELPER)
		restore_finish_stage(CR_STATE_RESTORE_PGID);

	return restore_one_task(current->pid.virt);
}

static int restore_root_task(struct pstree_item *init, struct cr_options *opts)
{
	int ret;
	struct sigaction act, old_act;

	ret = sigaction(SIGCHLD, NULL, &act);
	if (ret < 0) {
		pr_perror("sigaction() failed\n");
		return -1;
	}

	act.sa_flags |= SA_NOCLDSTOP | SA_SIGINFO | SA_RESTART;
	act.sa_sigaction = sigchld_handler;
	sigemptyset(&act.sa_mask);
	sigaddset(&act.sa_mask, SIGCHLD);

	ret = sigaction(SIGCHLD, &act, &old_act);
	if (ret < 0) {
		pr_perror("sigaction() failed\n");
		return -1;
	}

	/*
	 * FIXME -- currently we assume that all the tasks live
	 * in the same set of namespaces. This is done to debug
	 * the ns contents dumping/restoring. Need to revisit
	 * this later.
	 */

	if (init->pid.virt == INIT_PID) {
		if (!(opts->namespaces_flags & CLONE_NEWPID)) {
			pr_err("This process tree can be restored in a new pid namespace.\n");
			pr_err("crtools should be re-executed with --namespace pid\n");
			return -1;
		}
	} else	if (opts->namespaces_flags & CLONE_NEWPID) {
		pr_err("Can't restore pid namespace without the process init\n");
		return -1;
	}

	futex_set(&task_entries->nr_in_progress, task_entries->nr_tasks + task_entries->nr_helpers);

	ret = fork_with_pid(init, opts->namespaces_flags);
	if (ret < 0)
		return -1;

	pr_info("Wait until all tasks are forked\n");
	futex_wait_while_gt(&task_entries->nr_in_progress, 0);
	ret = (int)futex_get(&task_entries->nr_in_progress);
	if (ret < 0)
		goto out;

	futex_set_and_wake(&task_entries->nr_in_progress, task_entries->nr_tasks);
	futex_set_and_wake(&task_entries->start, CR_STATE_RESTORE_PGID);

	pr_info("Wait until all tasks restored pgid\n");
	futex_wait_while_gt(&task_entries->nr_in_progress, 0);
	ret = (int)futex_get(&task_entries->nr_in_progress);
	if (ret < 0)
		goto out;

	futex_set_and_wake(&task_entries->nr_in_progress, task_entries->nr_threads);
	futex_set_and_wake(&task_entries->start, CR_STATE_RESTORE);

	pr_info("Wait until all tasks are restored\n");
	futex_wait_while_gt(&task_entries->nr_in_progress, 0);
	ret = (int)futex_get(&task_entries->nr_in_progress);
	if (ret < 0)
		goto out;

	futex_set_and_wake(&task_entries->nr_in_progress, task_entries->nr_threads);
	futex_set_and_wake(&task_entries->start, CR_STATE_RESTORE_SIGCHLD);
	futex_wait_until(&task_entries->nr_in_progress, 0);

	/* Restore SIGCHLD here to skip SIGCHLD from a network sctip */
	ret = sigaction(SIGCHLD, &old_act, NULL);
	if (ret < 0) {
		pr_perror("sigaction() failed\n");
		goto out;
	}

	network_unlock();
out:
	if (ret < 0) {
		struct pstree_item *pi;<commit_msg>rst: Helper for switching restore stages<commit_after>#include <stdio.h>
#include <stdlib.h>
#include <signal.h>
#include <limits.h>
#include <unistd.h>
#include <errno.h>
#include <dirent.h>
#include <string.h>

#include <fcntl.h>
#include <grp.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <sys/vfs.h>
#include <sys/ptrace.h>
#include <sys/wait.h>
#include <sys/file.h>
#include <sys/shm.h>
#include <sys/mount.h>

#include <sched.h>

#include <sys/sendfile.h>

#include "compiler.h"
#include "types.h"

#include "image.h"
#include "util.h"
#include "log.h"
#include "syscall.h"
#include "restorer.h"
#include "sockets.h"
#include "sk-packet.h"
#include "lock.h"
#include "files.h"
#include "files-reg.h"
#include "pipes.h"
#include "fifo.h"
#include "sk-inet.h"
#include "eventfd.h"
#include "eventpoll.h"
#include "signalfd.h"
#include "proc_parse.h"
#include "restorer-blob.h"
#include "crtools.h"
#include "namespaces.h"
#include "shmem.h"
#include "mount.h"
#include "inotify.h"
#include "pstree.h"
#include "net.h"
#include "tty.h"

#include "protobuf.h"
#include "protobuf/sa.pb-c.h"
#include "protobuf/itimer.pb-c.h"
#include "protobuf/vma.pb-c.h"

static struct pstree_item *current;

static int restore_task_with_children(void *);
static int sigreturn_restore(pid_t pid, CoreEntry *core);
static int prepare_restorer_blob(void);

static LIST_HEAD(rst_vma_list);
static int rst_nr_vmas;
static void *premmapped_addr;
static unsigned long premmapped_len;

static int shmem_remap(void *old_addr, void *new_addr, unsigned long size)
{
	void *ret;

	ret = mremap(old_addr, size, size,
			MREMAP_FIXED | MREMAP_MAYMOVE, new_addr);
	if (new_addr != ret) {
		pr_perror("mremap failed");
		return -1;
	}

	return 0;
}

static int crtools_prepare_shared(void)
{
	if (prepare_shared_fdinfo())
		return -1;

	/* Connections are unlocked from crtools */
	if (collect_inet_sockets())
		return -1;

	if (tty_prep_fds())
		return -1;

	return 0;
}

static int root_prepare_shared(void)
{
	int ret = 0;
	struct pstree_item *pi;

	pr_info("Preparing info about shared resources\n");

	if (prepare_shmem_restore())
		return -1;

	if (prepare_shared_tty())
		return -1;

	if (prepare_shared_reg_files())
		return -1;

	if (collect_reg_files())
		return -1;

	if (collect_pipes())
		return -1;

	if (collect_fifo())
		return -1;

	if (collect_unix_sockets())
		return -1;

	if (collect_packet_sockets())
		return -1;

	if (collect_eventfd())
		return -1;

	if (collect_eventpoll())
		return -1;

	if (collect_signalfd())
		return -1;

	if (collect_inotify())
		return -1;

	if (collect_tty())
		return -1;

	for_each_pstree_item(pi) {
		if (pi->state == TASK_HELPER)
			continue;

		ret = prepare_shmem_pid(pi->pid.virt);
		if (ret < 0)
			break;

		ret = prepare_fd_pid(pi->pid.virt, pi->rst);
		if (ret < 0)
			break;
	}

	if (ret < 0)
		goto err;

	mark_pipe_master();

	ret = tty_setup_slavery();
	if (ret)
		goto err;

	ret = resolve_unix_peers();
	if (ret)
		goto err;

	ret = prepare_restorer_blob();
	if (ret)
		goto err;

	show_saved_shmems();
	show_saved_files();
err:
	return ret;
}

/* Map a private vma, if it is not mapped by a parrent yet */
static int map_private_vma(pid_t pid, struct vma_area *vma, void *tgt_addr,
			struct vma_area **pvma, struct list_head *pvma_list)
{
	int ret;
	void *addr, *paddr = NULL;
	unsigned long nr_pages;
	struct vma_area *p = *pvma;

	if (vma_entry_is(&vma->vma, VMA_FILE_PRIVATE)) {
		ret = get_filemap_fd(pid, &vma->vma);
		if (ret < 0) {
			pr_err("Can't fixup fd\n");
			return -1;
		}
		vma->vma.fd = ret;
		/* shmid will be used for a temporary address */
		vma->vma.shmid = 0;
	}

	nr_pages = vma_entry_len(&vma->vma) / PAGE_SIZE;
	vma->page_bitmap = xzalloc(BITS_TO_LONGS(nr_pages) * sizeof(long));
	if (vma->page_bitmap == NULL)
		return -1;

	list_for_each_entry_continue(p, pvma_list, list) {
		if (p->vma.start > vma->vma.start)
			 break;

		if (p->vma.end == vma->vma.end &&
		    p->vma.start == vma->vma.start) {
			pr_info("COW 0x%016lx-0x%016lx 0x%016lx vma\n",
				vma->vma.start, vma->vma.end, vma->vma.pgoff);
			paddr = (void *) vma_premmaped_start(&p->vma);
			break;
		}

	}

	*pvma = p;

	if (paddr == NULL) {
		pr_info("Map 0x%016lx-0x%016lx 0x%016lx vma\n",
			vma->vma.start, vma->vma.end, vma->vma.pgoff);

		addr = mmap(tgt_addr, vma_entry_len(&vma->vma),
				vma->vma.prot | PROT_WRITE,
				vma->vma.flags | MAP_FIXED,
				vma->vma.fd, vma->vma.pgoff);

		if (addr == MAP_FAILED) {
			pr_perror("Unable to map ANON_VMA");
			return -1;
		}
	} else {
		vma->ppage_bitmap = p->page_bitmap;

		addr = mremap(paddr, vma_area_len(vma), vma_area_len(vma),
				MREMAP_FIXED | MREMAP_MAYMOVE, tgt_addr);
		if (addr != tgt_addr) {
			pr_perror("Unable to remap a private vma");
			return -1;
		}

	}

	vma_premmaped_start(&(vma->vma)) = (unsigned long) addr;

	if (vma_entry_is(&vma->vma, VMA_FILE_PRIVATE))
		close(vma->vma.fd);

	return 0;
}

static int restore_priv_vma_content(pid_t pid)
{
	struct vma_area *vma;
	int fd, ret = 0;

	unsigned int nr_restored = 0;
	unsigned int nr_shared = 0;
	unsigned int nr_droped = 0;

	vma = list_first_entry(&rst_vma_list, struct vma_area, list);

	fd = open_image_ro(CR_FD_PAGES, pid);
	if (fd < 0)
		return -1;

	/*
	 * Read page contents.
	 */
	while (1) {
		u64 va, page_offset;
		char buf[PAGE_SIZE];
		void *p;

		ret = read(fd, &va, sizeof(va));
		if (!ret)
			break;

		if (ret != sizeof(va)) {
			pr_err("Bad mapping page size %d\n", ret);
			return -1;
		}

		BUG_ON(va < vma->vma.start);

		while (va >= vma->vma.end) {
			BUG_ON(vma->list.next == &rst_vma_list);
			vma = list_entry(vma->list.next, struct vma_area, list);
		}

		page_offset = (va - vma->vma.start) / PAGE_SIZE;

		set_bit(page_offset, vma->page_bitmap);
		if (vma->ppage_bitmap)
			clear_bit(page_offset, vma->ppage_bitmap);

		ret = read(fd, buf, PAGE_SIZE);
		if (ret != PAGE_SIZE) {
			pr_err("Can'r read mapping page %d\n", ret);
			return -1;
		}

		p = (void *) (va - vma->vma.start +
					vma_premmaped_start(&vma->vma));
		if (memcmp(p, buf, PAGE_SIZE) == 0) {
			nr_shared++;
			continue;
		}

		memcpy(p, buf, PAGE_SIZE);
		nr_restored++;
	}
	close(fd);

	/* Remove pages, which were not shared with a child */
	list_for_each_entry(vma, &rst_vma_list, list) {
		unsigned long size, i = 0;
		void *addr = (void *) vma_premmaped_start(&vma->vma);

		if (vma->ppage_bitmap == NULL)
			continue;

		size = vma_entry_len(&vma->vma) / PAGE_SIZE;
		while (1) {
			/* Find all pages, which are not shared with this child */
			i = find_next_bit(vma->ppage_bitmap, size, i);

			if ( i >= size)
				break;

			ret = madvise(addr + PAGE_SIZE * i,
						PAGE_SIZE, MADV_DONTNEED);
			if (ret < 0) {
				pr_perror("madvise failed\n");
				return -1;
			}
			i++;
			nr_droped++;
		}
	}

	pr_info("nr_restored_pages: %d\n", nr_restored);
	pr_info("nr_shared_pages:   %d\n", nr_shared);
	pr_info("nr_droped_pages:   %d\n", nr_droped);

	return 0;
}

static int read_vmas(int pid)
{
	int fd, ret = 0;
	LIST_HEAD(old);
	struct vma_area *pvma, *vma;
	unsigned long priv_size = 0;
	void *addr;

	void *old_premmapped_addr = NULL;
	unsigned long old_premmapped_len, pstart = 0;

	rst_nr_vmas = 0;
	list_replace_init(&rst_vma_list, &old);

	/* Skip errors, because a zombie doesn't have an image of vmas */
	fd = open_image_ro(CR_FD_VMAS, pid);
	if (fd < 0) {
		if (errno != ENOENT)
			ret = fd;
		goto out;
	}

	while (1) {
		struct vma_area *vma;
		VmaEntry *e;

		ret = -1;
		vma = alloc_vma_area();
		if (!vma)
			break;

		ret = pb_read_one_eof(fd, &e, PB_VMAS);
		if (ret <= 0)
			break;

		rst_nr_vmas++;
		list_add_tail(&vma->list, &rst_vma_list);

		if (e->fd != -1) {
			ret = -1;
			pr_err("Error in vma->fd setting (%Ld)\n",
					(unsigned long long)e->fd);
			break;
		}

		vma->vma = *e;
		vma_entry__free_unpacked(e, NULL);

		if (!vma_priv(&vma->vma))
			continue;

		priv_size += vma_area_len(vma);
	}

	/* Reserve a place for mapping private vma-s one by one */
	addr = mmap(NULL, priv_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
	if (addr == MAP_FAILED) {
		pr_perror("Unable to reserve memory");
		return -1;
	}

	old_premmapped_addr = premmapped_addr;
	old_premmapped_len = premmapped_len;
	premmapped_addr = addr;
	premmapped_len = priv_size;

	pvma = list_entry(&old, struct vma_area, list);

	list_for_each_entry(vma, &rst_vma_list, list) {
		if (pstart > vma->vma.start) {
			ret = -1;
			pr_err("VMA-s are not sorted in the image file\n");
			break;
		}
		pstart = vma->vma.start;

		if (!vma_priv(&vma->vma))
			continue;

		ret = map_private_vma(pid, vma, addr, &pvma, &old);
		if (ret < 0)
			break;

		addr += vma_area_len(vma);
	}

	if (ret == 0)
		ret = restore_priv_vma_content(pid);
	close(fd);

out:
	while (!list_empty(&old)) {
		vma = list_first_entry(&old, struct vma_area, list);
		list_del(&vma->list);
		xfree(vma);
	}

	if (old_premmapped_addr &&
	    munmap(old_premmapped_addr, old_premmapped_len)) {
		pr_perror("Unable to unmap %p(%lx)",
				old_premmapped_addr, old_premmapped_len);
		return -1;
	}


	return ret;
}

static int open_vmas(int pid, struct list_head *vmas)
{
	struct vma_area *vma;
	int ret = 0;

	list_for_each_entry(vma, vmas, list) {
		if (!(vma_entry_is(&vma->vma, VMA_AREA_REGULAR)))
			continue;

		pr_info("Opening 0x%016lx-0x%016lx 0x%016lx (%x) vma\n",
				vma->vma.start, vma->vma.end,
				vma->vma.pgoff, vma->vma.status);

		if (vma_entry_is(&vma->vma, VMA_AREA_SYSVIPC))
			ret = vma->vma.shmid;
		else if (vma_entry_is(&vma->vma, VMA_ANON_SHARED))
			ret = get_shmem_fd(pid, &vma->vma);
		else if (vma_entry_is(&vma->vma, VMA_FILE_SHARED))
			ret = get_filemap_fd(pid, &vma->vma);
		else if (vma_entry_is(&vma->vma, VMA_AREA_SOCKET))
			ret = get_socket_fd(pid, &vma->vma);
		else
			continue;

		if (ret < 0) {
			pr_err("Can't fixup fd\n");
			break;
		}

		pr_info("\t`- setting %d as mapping fd\n", ret);
		vma->vma.fd = ret;
	}

	return ret < 0 ? -1 : 0;
}

static int prepare_and_sigreturn(int pid, CoreEntry *core)
{
	int err;

	err = open_vmas(pid, &rst_vma_list);
	if (err)
		return err;

	return sigreturn_restore(pid, core);
}

static rt_sigaction_t sigchld_act;
static int prepare_sigactions(int pid)
{
	rt_sigaction_t act, oact;
	int fd_sigact;
	SaEntry *e;
	int sig;
	int ret = -1;

	fd_sigact = open_image_ro(CR_FD_SIGACT, pid);
	if (fd_sigact < 0)
		return -1;

	for (sig = 1; sig < SIGMAX; sig++) {
		if (sig == SIGKILL || sig == SIGSTOP)
			continue;

		ret = pb_read_one(fd_sigact, &e, PB_SIGACT);
		if (ret < 0)
			break;

		ASSIGN_TYPED(act.rt_sa_handler, e->sigaction);
		ASSIGN_TYPED(act.rt_sa_flags, e->flags);
		ASSIGN_TYPED(act.rt_sa_restorer, e->restorer);
		ASSIGN_TYPED(act.rt_sa_mask.sig[0], e->mask);

		sa_entry__free_unpacked(e, NULL);

		if (sig == SIGCHLD) {
			sigchld_act = act;
			continue;
		}
		/*
		 * A pure syscall is used, because glibc
		 * sigaction overwrites se_restorer.
		 */
		ret = sys_sigaction(sig, &act, &oact, sizeof(rt_sigset_t));
		if (ret == -1) {
			pr_err("%d: Can't restore sigaction: %m\n", pid);
			goto err;
		}
	}

err:
	close_safe(&fd_sigact);
	return ret;
}

static int pstree_wait_helpers()
{
	struct pstree_item *pi;

	list_for_each_entry(pi, &current->children, sibling) {
		int status, ret;

		if (pi->state != TASK_HELPER)
			continue;

		/* Check, that a helper completed. */
		ret = waitpid(pi->pid.virt, &status, 0);
		if (ret == -1) {
			if (errno == ECHILD)
				continue; /* It has been waited in sigchld_handler */
			pr_err("waitpid(%d) failed\n", pi->pid.virt);
			return -1;
		}
		if (!WIFEXITED(status) || WEXITSTATUS(status)) {
			pr_err("%d exited with non-zero code (%d,%d)\n", pi->pid.virt,
				WEXITSTATUS(status), WTERMSIG(status));
			return -1;
		}

	}

	return 0;
}


static int restore_one_alive_task(int pid, CoreEntry *core)
{
	pr_info("Restoring resources\n");

	if (pstree_wait_helpers())
		return -1;

	if (prepare_fds(current))
		return -1;

	if (prepare_fs(pid))
		return -1;

	if (prepare_sigactions(pid))
		return -1;

	log_closedir();

	return prepare_and_sigreturn(pid, core);
}

static void zombie_prepare_signals(void)
{
	sigset_t blockmask;
	int sig;
	struct sigaction act;

	sigfillset(&blockmask);
	sigprocmask(SIG_UNBLOCK, &blockmask, NULL);

	memset(&act, 0, sizeof(act));
	act.sa_handler = SIG_DFL;

	for (sig = 1; sig < SIGMAX; sig++)
		sigaction(sig, &act, NULL);
}

#define SIG_FATAL_MASK	(	\
		(1 << SIGHUP)	|\
		(1 << SIGINT)	|\
		(1 << SIGQUIT)	|\
		(1 << SIGILL)	|\
		(1 << SIGTRAP)	|\
		(1 << SIGABRT)	|\
		(1 << SIGIOT)	|\
		(1 << SIGBUS)	|\
		(1 << SIGFPE)	|\
		(1 << SIGKILL)	|\
		(1 << SIGUSR1)	|\
		(1 << SIGSEGV)	|\
		(1 << SIGUSR2)	|\
		(1 << SIGPIPE)	|\
		(1 << SIGALRM)	|\
		(1 << SIGTERM)	|\
		(1 << SIGXCPU)	|\
		(1 << SIGXFSZ)	|\
		(1 << SIGVTALRM)|\
		(1 << SIGPROF)	|\
		(1 << SIGPOLL)	|\
		(1 << SIGIO)	|\
		(1 << SIGSYS)	|\
		(1 << SIGUNUSED)|\
		(1 << SIGSTKFLT)|\
		(1 << SIGPWR)	 \
	)

static inline int sig_fatal(int sig)
{
	return (sig > 0) && (sig < SIGMAX) && (SIG_FATAL_MASK & (1 << sig));
}

struct task_entries *task_entries;

static int restore_one_fake(int pid)
{
	/* We should wait here, otherwise last_pid will be changed. */
	futex_wait_while(&task_entries->start, CR_STATE_FORKING);
	futex_wait_while(&task_entries->start, CR_STATE_RESTORE_PGID);
	return 0;
}

static int restore_one_zombie(int pid, int exit_code)
{
	pr_info("Restoring zombie with %d code\n", exit_code);

	if (task_entries != NULL) {
		restore_finish_stage(CR_STATE_RESTORE);
		zombie_prepare_signals();
		restore_finish_stage(CR_STATE_RESTORE_SIGCHLD);
	}

	if (exit_code & 0x7f) {
		int signr;

		signr = exit_code & 0x7F;
		if (!sig_fatal(signr)) {
			pr_warn("Exit with non fatal signal ignored\n");
			signr = SIGABRT;
		}

		if (kill(pid, signr) < 0)
			pr_perror("Can't kill myself, will just exit");

		exit_code = 0;
	}

	exit((exit_code >> 8) & 0x7f);

	/* never reached */
	BUG_ON(1);
	return -1;
}

static int check_core(CoreEntry *core)
{
	int ret = -1;

	if (core->mtype != CORE_ENTRY__MARCH__X86_64) {
		pr_err("Core march mismatch %d\n", (int)core->mtype);
		goto out;
	}

	if (!core->tc) {
		pr_err("Core task state data missed\n");
		goto out;
	}

	if (core->tc->task_state != TASK_DEAD) {
		if (!core->ids) {
			pr_err("Core IDS data missed for non-zombie\n");
			goto out;
		}

		if (!core->thread_info) {
			pr_err("Core info data missed for non-zombie\n");
			goto out;
		}
	}

	ret = 0;
out:
	return ret < 0 ? ret : 0;
}

static int restore_one_task(int pid)
{
	int fd, ret;
	CoreEntry *core;

	if (current->state == TASK_HELPER)
		return restore_one_fake(pid);

	fd = open_image_ro(CR_FD_CORE, pid);
	if (fd < 0)
		return -1;

	ret = pb_read_one(fd, &core, PB_CORE);
	close(fd);

	if (ret < 0)
		return -1;

	if (check_core(core)) {
		ret = -1;
		goto out;
	}

	switch ((int)core->tc->task_state) {
	case TASK_ALIVE:
		ret = restore_one_alive_task(pid, core);
		break;
	case TASK_DEAD:
		ret = restore_one_zombie(pid, core->tc->exit_code);
		break;
	default:
		pr_err("Unknown state in code %d\n", (int)core->tc->task_state);
		ret = -1;
		break;
	}

out:
	core_entry__free_unpacked(core, NULL);
	return ret;
}

/* All arguments should be above stack, because it grows down */
struct cr_clone_arg {
	char stack[PAGE_SIZE];
	char stack_ptr[0];
	struct pstree_item *item;
	unsigned long clone_flags;
	int fd;
};

static void write_pidfile(char *pfname, int pid)
{
	int fd;

	fd = open(pfname, O_WRONLY | O_TRUNC | O_CREAT, 0600);
	if (fd == -1) {
		pr_perror("Can't open %s", pfname);
		kill(pid, SIGKILL);
		return;
	}

	dprintf(fd, "%d", pid);
	close(fd);
}

static inline int fork_with_pid(struct pstree_item *item, unsigned long ns_clone_flags)
{
	int ret = -1;
	struct cr_clone_arg ca;
	pid_t pid = item->pid.virt;

	pr_info("Forking task with %d pid (flags 0x%lx)\n", pid, ns_clone_flags);

	ca.item = item;
	ca.clone_flags = ns_clone_flags;

	if (!(ca.clone_flags & CLONE_NEWPID)) {
		char buf[32];

		ca.fd = open(LAST_PID_PATH, O_RDWR);
		if (ca.fd < 0) {
			pr_perror("%d: Can't open %s", pid, LAST_PID_PATH);
			goto err;
		}

		if (flock(ca.fd, LOCK_EX)) {
			close(ca.fd);
			pr_perror("%d: Can't lock %s", pid, LAST_PID_PATH);
			goto err;
		}

		snprintf(buf, sizeof(buf), "%d", pid - 1);
		if (write_img_buf(ca.fd, buf, strlen(buf)))
			goto err_unlock;
	} else {
		ca.fd = -1;
		BUG_ON(pid != INIT_PID);
	}

	if (ca.clone_flags & CLONE_NEWNET)
		/*
		 * When restoring a net namespace we need to communicate
		 * with the original (i.e. -- init) one. Thus, prepare for
		 * that before we leave the existing namespaces.
		 */
		if (netns_pre_create())
			goto err_unlock;

	ret = clone(restore_task_with_children, ca.stack_ptr,
			ca.clone_flags | SIGCHLD, &ca);

	if (ret < 0)
		pr_perror("Can't fork for %d", pid);

	if (ca.clone_flags & CLONE_NEWPID)
		item->pid.real = ret;

	if (opts.pidfile && root_item == item)
		write_pidfile(opts.pidfile, ret);

err_unlock:
	if (ca.fd >= 0) {
		if (flock(ca.fd, LOCK_UN))
			pr_perror("%d: Can't unlock %s", pid, LAST_PID_PATH);

		close(ca.fd);
	}
err:
	return ret;
}

static void sigchld_handler(int signal, siginfo_t *siginfo, void *data)
{
	struct pstree_item *pi;
	pid_t pid = siginfo->si_pid;
	int status;
	int exit;

	exit = siginfo->si_code & CLD_EXITED;
	status = siginfo->si_status;
	if (!current || status)
		goto err;

	/* Skip a helper if it was completed successfully */
	while (pid) {
		pid = waitpid(-1, &status, WNOHANG);
		if (pid <= 0)
			return;

		exit = WIFEXITED(status);
		status = exit ? WEXITSTATUS(status) : WTERMSIG(status);
		if (status)
			break;

		list_for_each_entry(pi, &current->children, sibling) {
			if (pi->state != TASK_HELPER)
				continue;
			if (pi->pid.virt == siginfo->si_pid)
				break;
		}

		if (&pi->sibling == &current->children)
			break; /* The process is not a helper */
	}

err:
	if (exit)
		pr_err("%d exited, status=%d\n", pid, status);
	else
		pr_err("%d killed by signal %d\n", pid, status);

	futex_abort_and_wake(&task_entries->nr_in_progress);
}

/*
 * FIXME Din't fail on xid restore failure. MySQL uses runaway
 * pgid and sid and there's nothing we can do about it yet :(
 */

static void xid_fail(void)
{
	exit(1);
}

static void restore_sid(void)
{
	pid_t sid;

	/*
	 * SID can only be reset to pid or inherited from parent.
	 * Thus we restore it right here to let our kids inherit
	 * one in case they need it.
	 *
	 * PGIDs are restored late when all tasks are forked and
	 * we can call setpgid() on custom values.
	 */

	if (current->pid.virt == current->sid) {
		pr_info("Restoring %d to %d sid\n", current->pid.virt, current->sid);
		sid = setsid();
		if (sid != current->sid) {
			pr_perror("Can't restore sid (%d)", sid);
			xid_fail();
		}
	} else {
		sid = getsid(getpid());
		if (sid != current->sid) {
			/* Skip the root task if it's not init */
			if (current == root_item && root_item->pid.virt != INIT_PID)
				return;
			pr_err("Requested sid %d doesn't match inherited %d\n",
					current->sid, sid);
			xid_fail();
		}
	}
}

static void restore_pgid(void)
{
	pid_t pgid;

	pr_info("Restoring %d to %d pgid\n", current->pid.virt, current->pgid);

	pgid = getpgrp();
	if (current->pgid == pgid)
		return;

	pr_info("\twill call setpgid, mine pgid is %d\n", pgid);
	if (setpgid(0, current->pgid) != 0) {
		pr_perror("Can't restore pgid (%d/%d->%d)", current->pid.virt, pgid, current->pgid);
		xid_fail();
	}
}

static int mount_proc(void)
{
	int ret;
	char proc_mountpoint[PATH_MAX];

	snprintf(proc_mountpoint, sizeof(proc_mountpoint), "crtools-proc.XXXXXX");
	if (mkdtemp(proc_mountpoint) == NULL) {
		pr_perror("mkdtemp failed %s", proc_mountpoint);
		return -1;
	}

	pr_info("Mount procfs in %s\n", proc_mountpoint);
	if (mount("proc", proc_mountpoint, "proc", MS_MGC_VAL, NULL)) {
		pr_perror("mount failed");
		ret = -1;
		goto out_rmdir;
	}

	ret = set_proc_mountpoint(proc_mountpoint);

	if (umount2(proc_mountpoint, MNT_DETACH) == -1) {
		pr_perror("Can't umount %s", proc_mountpoint);
		return -1;
	}

out_rmdir:
	if (rmdir(proc_mountpoint) == -1) {
		pr_perror("Can't remove %s", proc_mountpoint);
		return -1;
	}

	return ret;
}

static int restore_task_with_children(void *_arg)
{
	struct cr_clone_arg *ca = _arg;
	struct pstree_item *child;
	pid_t pid;
	int ret;
	sigset_t blockmask;

	close_safe(&ca->fd);

	current = ca->item;

	pid = getpid();
	if (current->pid.virt != pid) {
		pr_err("Pid %d do not match expected %d\n", pid, current->pid.virt);
		exit(-1);
	}

	ret = log_init_by_pid();
	if (ret < 0)
		exit(1);

	/* Restore root task */
	if (current->parent == NULL) {
		if (collect_mount_info())
			exit(-1);

		if (prepare_namespace(current->pid.virt, ca->clone_flags))
			exit(-1);

		/*
		 * We need non /proc proc mount for restoring pid and mount
		 * namespaces and do not care for the rest of the cases.
		 * Thus -- mount proc at custom location for any new namespace
		 */
		if (mount_proc())
			exit(-1);

		if (root_prepare_shared())
			exit(-1);
	}

	/*
	 * The block mask will be restored in sigresturn.
	 *
	 * TODO: This code should be removed, when a freezer will be added.
	 */
	sigfillset(&blockmask);
	sigdelset(&blockmask, SIGCHLD);
	ret = sigprocmask(SIG_BLOCK, &blockmask, NULL);
	if (ret) {
		pr_perror("%d: Can't block signals", current->pid.virt);
		exit(1);
	}

	if (read_vmas(pid))
		exit(1);

	pr_info("Restoring children:\n");
	list_for_each_entry(child, &current->children, sibling) {
		if (!restore_before_setsid(child))
			continue;

		BUG_ON(child->born_sid != -1 && getsid(getpid()) != child->born_sid);

		ret = fork_with_pid(child, 0);
		if (ret < 0)
			exit(1);
	}

	restore_sid();

	pr_info("Restoring children:\n");
	list_for_each_entry(child, &current->children, sibling) {
		if (restore_before_setsid(child))
			continue;
		ret = fork_with_pid(child, 0);
		if (ret < 0)
			exit(1);
	}

	if (current->pgid == current->pid.virt)
		restore_pgid();

	restore_finish_stage(CR_STATE_FORKING);

	if (current->pgid != current->pid.virt)
		restore_pgid();

	if (current->state != TASK_HELPER)
		restore_finish_stage(CR_STATE_RESTORE_PGID);

	return restore_one_task(current->pid.virt);
}

static inline int stage_participants(int next_stage)
{
	switch (next_stage) {
	case CR_STATE_FORKING:
		return task_entries->nr_tasks + task_entries->nr_helpers;
	case CR_STATE_RESTORE_PGID:
		return task_entries->nr_tasks;
	case CR_STATE_RESTORE:
	case CR_STATE_RESTORE_SIGCHLD:
		return task_entries->nr_threads;
	}

	BUG();
	return -1;
}

static int restore_switch_stage(int next_stage)
{
	int ret;
	futex_t *np = &task_entries->nr_in_progress;

	futex_wait_while_gt(np, 0);
	ret = (int)futex_get(np);
	if (ret < 0)
		return ret;

	futex_set_and_wake(np, stage_participants(next_stage));
	futex_set_and_wake(&task_entries->start, next_stage);
	return 0;
}

static int restore_root_task(struct pstree_item *init, struct cr_options *opts)
{
	int ret;
	struct sigaction act, old_act;

	ret = sigaction(SIGCHLD, NULL, &act);
	if (ret < 0) {
		pr_perror("sigaction() failed\n");
		return -1;
	}

	act.sa_flags |= SA_NOCLDSTOP | SA_SIGINFO | SA_RESTART;
	act.sa_sigaction = sigchld_handler;
	sigemptyset(&act.sa_mask);
	sigaddset(&act.sa_mask, SIGCHLD);

	ret = sigaction(SIGCHLD, &act, &old_act);
	if (ret < 0) {
		pr_perror("sigaction() failed\n");
		return -1;
	}

	/*
	 * FIXME -- currently we assume that all the tasks live
	 * in the same set of namespaces. This is done to debug
	 * the ns contents dumping/restoring. Need to revisit
	 * this later.
	 */

	if (init->pid.virt == INIT_PID) {
		if (!(opts->namespaces_flags & CLONE_NEWPID)) {
			pr_err("This process tree can be restored in a new pid namespace.\n");
			pr_err("crtools should be re-executed with --namespace pid\n");
			return -1;
		}
	} else	if (opts->namespaces_flags & CLONE_NEWPID) {
		pr_err("Can't restore pid namespace without the process init\n");
		return -1;
	}

	futex_set(&task_entries->nr_in_progress, stage_participants(CR_STATE_FORKING));

	ret = fork_with_pid(init, opts->namespaces_flags);
	if (ret < 0)
		return -1;

	pr_info("Wait until all tasks are forked\n");
	ret = restore_switch_stage(CR_STATE_RESTORE_PGID);
	if (ret < 0)
		goto out;


	pr_info("Wait until all tasks restored pgid\n");
	ret = restore_switch_stage(CR_STATE_RESTORE);
	if (ret < 0)
		goto out;

	pr_info("Wait until all tasks are restored\n");
	ret = restore_switch_stage(CR_STATE_RESTORE_SIGCHLD);
	if (ret < 0)
		goto out;

	futex_wait_until(&task_entries->nr_in_progress, 0);

	/* Restore SIGCHLD here to skip SIGCHLD from a network sctip */
	ret = sigaction(SIGCHLD, &old_act, NULL);
	if (ret < 0) {
		pr_perror("sigaction() failed\n");
		goto out;
	}

	network_unlock();
out:
	if (ret < 0) {
		struct pstree_item *pi;
{'fraction_non_alphanumeric': 0.15403011694128615, 'fraction_numerical': 0.009707950791431061, 'mean_word_length': 3.7655012555534095, 'pattern_counts': {'":': 0, '<': 224, '<?xml version=': 0, '>': 380, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3248605', 'n_tokens_mistral': 22254, 'n_tokens_neox': 20499, 'n_words': 4728}