<commit_before>package bamboo.task;

import bamboo.core.Config;
import bamboo.core.Db;
import bamboo.core.DbPool;
import bamboo.core.Scrub;
import bamboo.io.HeritrixJob;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.PosixFileAttributes;
import java.util.List;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

public class ImportJob {
	final Config config;
	final DbPool dbPool;
	final long crawlId;

	private HeritrixJob heritrixJob;
	private List<Path> launches;

	public ImportJob(Config config, DbPool dbPool, long crawlId) {
		this.config = config;
		this.dbPool = dbPool;
		this.crawlId = crawlId;
	}
	
	public void run()  {
		Db.Crawl crawl;
		Db.CrawlSeries series;

		try (Db db = dbPool.take()) {
			crawl = db.findCrawl(crawlId);
			if (crawl == null)
				throw new RuntimeException("Crawl " + crawlId + " not found");
			if (crawl.state != Db.IMPORTING) {
				return; // sanity check
			}
			if (crawl.crawlSeriesId == null)
				throw new RuntimeException("TODO: implement imports without a series");

			series = db.findCrawlSeriesById(crawl.crawlSeriesId);
			if (series == null)
				throw new RuntimeException("Couldn't find crawl series " + crawl.crawlSeriesId);
		}

		try {
			heritrixJob = HeritrixJob.byName(config.getHeritrixJobs(), crawl.name);
			heritrixJob.checkSuitableForArchiving();

			Path dest = allocateCrawlPath(crawl, series);
			Path warcsDir = dest.resolve("warcs");
			Files.createDirectory(warcsDir);
			copyWarcs(heritrixJob.warcs().collect(Collectors.toList()), warcsDir);
			constructCrawlBundle(heritrixJob.dir(), dest);

			try (Db db = dbPool.take()) {
				db.updateCrawlState(crawl.id, Db.ARCHIVED);
			}<commit_msg>Don't allocate a new crawl path if one already exists<commit_after>package bamboo.task;

import bamboo.core.Config;
import bamboo.core.Db;
import bamboo.core.DbPool;
import bamboo.core.Scrub;
import bamboo.io.HeritrixJob;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.PosixFileAttributes;
import java.util.List;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

public class ImportJob {
	final Config config;
	final DbPool dbPool;
	final long crawlId;

	private HeritrixJob heritrixJob;
	private List<Path> launches;

	public ImportJob(Config config, DbPool dbPool, long crawlId) {
		this.config = config;
		this.dbPool = dbPool;
		this.crawlId = crawlId;
	}
	
	public void run()  {
		Db.Crawl crawl;
		Db.CrawlSeries series;

		try (Db db = dbPool.take()) {
			crawl = db.findCrawl(crawlId);
			if (crawl == null)
				throw new RuntimeException("Crawl " + crawlId + " not found");
			if (crawl.state != Db.IMPORTING) {
				return; // sanity check
			}
			if (crawl.crawlSeriesId == null)
				throw new RuntimeException("TODO: implement imports without a series");

			series = db.findCrawlSeriesById(crawl.crawlSeriesId);
			if (series == null)
				throw new RuntimeException("Couldn't find crawl series " + crawl.crawlSeriesId);
		}

		try {
			heritrixJob = HeritrixJob.byName(config.getHeritrixJobs(), crawl.name);
			heritrixJob.checkSuitableForArchiving();

			Path dest;

			if (crawl.path != null) {
				dest = crawl.path;
			} else {
				dest = allocateCrawlPath(crawl, series);
			}

			Path warcsDir = dest.resolve("warcs");

			if (!Files.exists(warcsDir)) {
				Files.createDirectory(warcsDir);
			}

			copyWarcs(heritrixJob.warcs().collect(Collectors.toList()), warcsDir);
			constructCrawlBundle(heritrixJob.dir(), dest);

			try (Db db = dbPool.take()) {
				db.updateCrawlState(crawl.id, Db.ARCHIVED);
			}
{'fraction_non_alphanumeric': 0.11782401604412134, 'fraction_numerical': 0.0, 'mean_word_length': 4.32, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4203615', 'n_tokens_mistral': 1564, 'n_tokens_neox': 1330, 'n_words': 296}