<commit_before>import path from 'path';
import easydate from 'easydate';
import fsWriteStreamAtomic from 'fs-write-stream-atomic';
import getRes from 'get-res';
import logSymbols from 'log-symbols';
import memoizeAsync from 'memoize-async';
import mkdirp from 'mkdirp';
import rimraf from 'rimraf';
import screenshotStream from 'screenshot-stream';
import viewportList from 'viewport-list';
import protocolify from 'protocolify';
import arrayUniq from 'array-uniq';
import filenamifyUrl from 'filenamify-url';
import template from 'lodash.template';
import pify from 'pify';
import plur from 'plur';

/**
 * Fetch ten most popular resolutions
 *
 * @param {String} url
 * @param {Object} options
 * @api private
 */

export async function resolution(url, options) {
	const items = await pify(memoizeAsync(getRes))();

	for (const item of items) {
		this.sizes.push(item.item);
		this.items.push(this.create(url, item.item, options));
	}
}

/**
 * Fetch keywords
 *
 * @param {Object} obj
 * @param {Object} options
 */

export async function viewport(obj, options) {
	const items = await pify(memoizeAsync(viewportList))(obj.keywords);

	for (const item of items) {
		this.sizes.push(item.size);
		obj.sizes.push(item.size);
	}

	for (const size of arrayUniq(obj.sizes)) {
		this.items.push(this.create(obj.url, size, options));
	}
}

/**
 * Save an array of streams to files
 *
 * @param {Array} streams
 * @api private
 */

export async function save(streams) {
	const files = [];

	async function end() {
		return await* files.map(file => pify(rimraf)(file));
	}

	process.on('SIGINT', async () => process.exit(await end()));

	return await* streams.map(stream => {
		return new Promise(async (resolve, reject) => {
			await pify(mkdirp)(this.dest());

			const dest = path.join(this.dest(), stream.filename);
			const write = fsWriteStreamAtomic(dest);

			files.push(write.__atomicTmp);

			stream.on('warn', this.emit.bind(this, 'warn'));
			stream.on('error', err => end().then(reject(err)));

			write.on('finish', resolve);
			write.on('error', err => end().then(reject(err)));

			stream.pipe(write);
		});
	});
}

/**
 * Create a pageres stream
 *
 * @param {String} uri
 * @param {String} size
 * @param {Object} options
 * @api private
 */
<commit_msg>fix linting<commit_after>import path from 'path';
import easydate from 'easydate';
import fsWriteStreamAtomic from 'fs-write-stream-atomic';
import getRes from 'get-res';
import logSymbols from 'log-symbols';
import memoizeAsync from 'memoize-async';
import mkdirp from 'mkdirp';
import rimraf from 'rimraf';
import screenshotStream from 'screenshot-stream';
import viewportList from 'viewport-list';
import protocolify from 'protocolify';
import arrayUniq from 'array-uniq';
import filenamifyUrl from 'filenamify-url';
import template from 'lodash.template';
import pify from 'pify';
import plur from 'plur';

/**
 * Fetch ten most popular resolutions
 *
 * @param {String} url
 * @param {Object} options
 * @api private
 */

export async function resolution(url, options) {
	const items = await pify(memoizeAsync(getRes))();

	for (const item of items) {
		this.sizes.push(item.item);
		this.items.push(this.create(url, item.item, options));
	}
}

/**
 * Fetch keywords
 *
 * @param {Object} obj
 * @param {Object} options
 */

export async function viewport(obj, options) {
	const items = await pify(memoizeAsync(viewportList))(obj.keywords);

	for (const item of items) {
		this.sizes.push(item.size);
		obj.sizes.push(item.size);
	}

	for (const size of arrayUniq(obj.sizes)) {
		this.items.push(this.create(obj.url, size, options));
	}
}

/**
 * Save an array of streams to files
 *
 * @param {Array} streams
 * @api private
 */

export async function save(streams) {
	const files = [];

	async function end() {
		return await* files.map(file => pify(rimraf)(file));
	}

	process.on('SIGINT', async () => process.exit(await end()));

	return await* streams.map(stream =>
		new Promise(async (resolve, reject) => {
			await pify(mkdirp)(this.dest());

			const dest = path.join(this.dest(), stream.filename);
			const write = fsWriteStreamAtomic(dest);

			files.push(write.__atomicTmp);

			stream.on('warn', this.emit.bind(this, 'warn'));
			stream.on('error', err => end().then(reject(err)));

			write.on('finish', resolve);
			write.on('error', err => end().then(reject(err)));

			stream.pipe(write);
		})
	);
}

/**
 * Create a pageres stream
 *
 * @param {String} uri
 * @param {String} size
 * @param {Object} options
 * @api private
 */

{'fraction_non_alphanumeric': 0.15519149878237767, 'fraction_numerical': 0.0, 'mean_word_length': 4.456521739130435, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4170382', 'n_tokens_mistral': 1576, 'n_tokens_neox': 1487, 'n_words': 402}