<commit_before>import * as _ from 'lodash';
import * as EventEmitter from 'events';
import * as dockerProgress from 'docker-progress';
import * as Docker from 'dockerode';
import * as path from 'path';
import * as streamModule from 'stream';
import * as Bluebird from 'bluebird';
import * as tarfs from 'tar-fs';
import {
	promises as fs,
	constants,
	createReadStream,
	createWriteStream,
} from 'fs';
import * as getPort from 'get-port';
import * as os from 'os';
import * as tmp from 'tmp-promise';
import * as unzipper from 'unzipper';
import * as archiver from 'archiver';
import * as compareVersions from 'compare-versions';
import * as request from 'request-promise';
import { promisify } from 'util';
import getFolderSize = require('get-folder-size');
import { Application, PineFilter, Release } from 'balena-sdk';
import { DirOptions } from 'tmp';
const getFolderSizeAsync: (arg1: string) => Promise<number> =
	promisify(getFolderSize);

const preload = module.exports;

const { R_OK, W_OK } = constants;

const DOCKER_TMPDIR = '/docker_tmpdir';
const DOCKER_IMAGE_TAG = 'balena/balena-preload';
const DISK_IMAGE_PATH_IN_DOCKER = '/img/balena.img';
const SPLASH_IMAGE_PATH_IN_DOCKER = '/img/balena-logo.png';
const DOCKER_STEP_RE = /Step (\d+)\/(\d+)/;
const CONCURRENT_REQUESTS_TO_REGISTRY = 10;

const GRAPHDRIVER_ERROR =
	'Error starting daemon: error initializing graphdriver: driver not supported';
const OVERLAY_MODULE_MESSAGE =
	'You need to load the "overlay" module to be able to preload this image: run "sudo modprobe overlay".';
const DOCKERD_USES_OVERLAY = '--storage-driver=overlay2';

const PARTITION_NAMES = ['resin-boot', 'resin-rootA', 'resin-data'];
const SUPERVISOR_USER_AGENT =
	'Supervisor/v6.6.0 (Linux; Resin OS 2.12.3; prod)';

const FLASH_EDISON_FILENAME = 'FlashEdison.json';
const EDISON_PARTITION_FILE_KEYS = {
	'resin-boot': 'boot_file',
	'resin-rootA': 'rootfs_file',
	'resin-data': 'resin-data_file',
};

const getEdisonPartitions = async (edisonFolder) => {
	// The replace is needed because this file contains new lines in strings, which is not valid JSON.
	const data = JSON.parse(
		(
			await fs.readFile(path.join(edisonFolder, FLASH_EDISON_FILENAME), 'utf8')
		).replace(/\n/g, ''),
	);
	const parameters = data.flash.parameters;
	const result = {};
	PARTITION_NAMES.forEach((name) => {
		result[name] = {
			file: path.join(
				edisonFolder,
				parameters[EDISON_PARTITION_FILE_KEYS[name]].value,
			),
			image: `/img/${name}`,
		};
	});
	return result;
};

class BufferBackedWritableStream extends streamModule.Writable {
	chunks: Buffer[] = [];

	_write(chunk, _enc, next) {
		this.chunks.push(chunk);
		next();
	}

	getData() {
		return Buffer.concat(this.chunks);
	}
}

function setBindMount(
	hostConfig: Docker.HostConfig,
	mounts: Array<[string, string]>,
	dockerApiVersion: string,
) {
	if (compareVersions(dockerApiVersion, '1.25') >= 0) {
		hostConfig.Mounts = mounts.map(([source, target]) => ({
			Source: path.resolve(source),
			Target: target,
			Type: 'bind',
			Consistency: 'delegated',
		}));
	} else {
		hostConfig.Binds = mounts.map(
			([source, target]) => `${path.resolve(source)}:${target}`,
		);
	}
}

const zipContainsFiles = (archive, files) => {
	// FIXME: read from the zip directory instead of reading the whole file
	// This is broken in unzipper for now: an invalid zip file will throw an
	// uncaught error event.
	const filesCopy = files.slice();
	return new Promise((resolve, reject) => {
		createReadStream(archive)
			.on('error', reject)
			.pipe(unzipper.Parse())
			.on('error', reject)
			.on('entry', function (entry) {
				_.pull(filesCopy, entry.path);
				entry.autodrain();
			})
			.on('finish', () => {
				resolve(filesCopy.length === 0);
			});
	});
};

const isEdisonZipArchive = async (file) => {
	try {
		await zipContainsFiles(file, [FLASH_EDISON_FILENAME]);
		return true;
	} catch {
		return false;
	}
};

type Layer = {
	digest: any;
	size: number;
};

type Manifest = {
	manifest: {
		layers: Layer[];
	};
	imageLocation: string;
};

type Image = {
	is_stored_at__image_location: string;
	image_size: number;
};

const createContainer = async (
	docker: Docker,
	image: string,
	splashImage: string,
	dockerPort: number,
	proxy: string,
	edisonFolder: string,
) => {
	const mounts: Array<[string, string]> = [];
	const version = await docker.version();
	if (os.platform() === 'linux') {
		// In some situations, devices created by `losetup -f` in the container only appear on the host and not in the container.
		// See https://github.com/balena-io/balena-cli/issues/1008
		mounts.push(['/dev', '/dev']);
	}
	if (splashImage) {
		mounts.push([splashImage, SPLASH_IMAGE_PATH_IN_DOCKER]);
	}

	const env = [
		`HTTP_PROXY=${proxy || ''}`,
		`HTTPS_PROXY=${proxy || ''}`,
		`DOCKER_PORT=${dockerPort || ''}`,
		`DOCKER_TMPDIR=${DOCKER_TMPDIR}`,
	];

	if (edisonFolder) {
		const partitions = getEdisonPartitions(edisonFolder);
		env.push(`PARTITIONS=${JSON.stringify(partitions)}`);
		PARTITION_NAMES.forEach((name) => {
			const part = partitions[name];
			mounts.push([part.file, part.image]);
		});
	} else {
		mounts.push([image, DISK_IMAGE_PATH_IN_DOCKER]);
	}
	const containerOptions: Docker.ContainerCreateOptions = {
		Image: DOCKER_IMAGE_TAG,
		name: preload.CONTAINER_NAME,
		AttachStdout: true,
		AttachStderr: true,
		OpenStdin: true,
		Env: env,
		Volumes: {
			[DOCKER_TMPDIR]: {},
		},
		HostConfig: {
			Privileged: true,
		},<commit_msg>Fix missing 'await' for getEdisonPartitions()<commit_after>import * as _ from 'lodash';
import * as EventEmitter from 'events';
import * as dockerProgress from 'docker-progress';
import * as Docker from 'dockerode';
import * as path from 'path';
import * as streamModule from 'stream';
import * as Bluebird from 'bluebird';
import * as tarfs from 'tar-fs';
import {
	promises as fs,
	constants,
	createReadStream,
	createWriteStream,
} from 'fs';
import * as getPort from 'get-port';
import * as os from 'os';
import * as tmp from 'tmp-promise';
import * as unzipper from 'unzipper';
import * as archiver from 'archiver';
import * as compareVersions from 'compare-versions';
import * as request from 'request-promise';
import { promisify } from 'util';
import getFolderSize = require('get-folder-size');
import { Application, PineFilter, Release } from 'balena-sdk';
import { DirOptions } from 'tmp';
const getFolderSizeAsync: (arg1: string) => Promise<number> =
	promisify(getFolderSize);

const preload = module.exports;

const { R_OK, W_OK } = constants;

const DOCKER_TMPDIR = '/docker_tmpdir';
const DOCKER_IMAGE_TAG = 'balena/balena-preload';
const DISK_IMAGE_PATH_IN_DOCKER = '/img/balena.img';
const SPLASH_IMAGE_PATH_IN_DOCKER = '/img/balena-logo.png';
const DOCKER_STEP_RE = /Step (\d+)\/(\d+)/;
const CONCURRENT_REQUESTS_TO_REGISTRY = 10;

const GRAPHDRIVER_ERROR =
	'Error starting daemon: error initializing graphdriver: driver not supported';
const OVERLAY_MODULE_MESSAGE =
	'You need to load the "overlay" module to be able to preload this image: run "sudo modprobe overlay".';
const DOCKERD_USES_OVERLAY = '--storage-driver=overlay2';

const PARTITION_NAMES = ['resin-boot', 'resin-rootA', 'resin-data'];
const SUPERVISOR_USER_AGENT =
	'Supervisor/v6.6.0 (Linux; Resin OS 2.12.3; prod)';

const FLASH_EDISON_FILENAME = 'FlashEdison.json';
const EDISON_PARTITION_FILE_KEYS = {
	'resin-boot': 'boot_file',
	'resin-rootA': 'rootfs_file',
	'resin-data': 'resin-data_file',
};

async function getEdisonPartitions(
	edisonFolder,
): Promise<{ [name: string]: { file: string; image: string } }> {
	// The replace is needed because this file contains new lines in strings, which is not valid JSON.
	const data = JSON.parse(
		(
			await fs.readFile(path.join(edisonFolder, FLASH_EDISON_FILENAME), 'utf8')
		).replace(/\n/g, ''),
	);
	const parameters = data.flash.parameters;
	const result = {};
	PARTITION_NAMES.forEach((name) => {
		result[name] = {
			file: path.join(
				edisonFolder,
				parameters[EDISON_PARTITION_FILE_KEYS[name]].value,
			),
			image: `/img/${name}`,
		};
	});
	return result;
}

class BufferBackedWritableStream extends streamModule.Writable {
	chunks: Buffer[] = [];

	_write(chunk, _enc, next) {
		this.chunks.push(chunk);
		next();
	}

	getData() {
		return Buffer.concat(this.chunks);
	}
}

function setBindMount(
	hostConfig: Docker.HostConfig,
	mounts: Array<[string, string]>,
	dockerApiVersion: string,
) {
	if (compareVersions(dockerApiVersion, '1.25') >= 0) {
		hostConfig.Mounts = mounts.map(([source, target]) => ({
			Source: path.resolve(source),
			Target: target,
			Type: 'bind',
			Consistency: 'delegated',
		}));
	} else {
		hostConfig.Binds = mounts.map(
			([source, target]) => `${path.resolve(source)}:${target}`,
		);
	}
}

const zipContainsFiles = (archive, files) => {
	// FIXME: read from the zip directory instead of reading the whole file
	// This is broken in unzipper for now: an invalid zip file will throw an
	// uncaught error event.
	const filesCopy = files.slice();
	return new Promise((resolve, reject) => {
		createReadStream(archive)
			.on('error', reject)
			.pipe(unzipper.Parse())
			.on('error', reject)
			.on('entry', function (entry) {
				_.pull(filesCopy, entry.path);
				entry.autodrain();
			})
			.on('finish', () => {
				resolve(filesCopy.length === 0);
			});
	});
};

const isEdisonZipArchive = async (file) => {
	try {
		await zipContainsFiles(file, [FLASH_EDISON_FILENAME]);
		return true;
	} catch {
		return false;
	}
};

type Layer = {
	digest: any;
	size: number;
};

type Manifest = {
	manifest: {
		layers: Layer[];
	};
	imageLocation: string;
};

type Image = {
	is_stored_at__image_location: string;
	image_size: number;
};

const createContainer = async (
	docker: Docker,
	image: string,
	splashImage: string,
	dockerPort: number,
	proxy: string,
	edisonFolder: string,
) => {
	const mounts: Array<[string, string]> = [];
	const version = await docker.version();
	if (os.platform() === 'linux') {
		// In some situations, devices created by `losetup -f` in the container only appear on the host and not in the container.
		// See https://github.com/balena-io/balena-cli/issues/1008
		mounts.push(['/dev', '/dev']);
	}
	if (splashImage) {
		mounts.push([splashImage, SPLASH_IMAGE_PATH_IN_DOCKER]);
	}

	const env = [
		`HTTP_PROXY=${proxy || ''}`,
		`HTTPS_PROXY=${proxy || ''}`,
		`DOCKER_PORT=${dockerPort || ''}`,
		`DOCKER_TMPDIR=${DOCKER_TMPDIR}`,
	];

	if (edisonFolder) {
		const partitions = await getEdisonPartitions(edisonFolder);
		env.push(`PARTITIONS=${JSON.stringify(partitions)}`);
		PARTITION_NAMES.forEach((name) => {
			const part = partitions[name];
			mounts.push([part.file, part.image]);
		});
	} else {
		mounts.push([image, DISK_IMAGE_PATH_IN_DOCKER]);
	}
	const containerOptions: Docker.ContainerCreateOptions = {
		Image: DOCKER_IMAGE_TAG,
		name: preload.CONTAINER_NAME,
		AttachStdout: true,
		AttachStderr: true,
		OpenStdin: true,
		Env: env,
		Volumes: {
			[DOCKER_TMPDIR]: {},
		},
		HostConfig: {
			Privileged: true,
		},
{'fraction_non_alphanumeric': 0.14942841589548175, 'fraction_numerical': 0.003810560696788242, 'mean_word_length': 4.539195979899498, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 33, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6366463', 'n_tokens_mistral': 4175, 'n_tokens_neox': 3886, 'n_words': 934}