<commit_before>"use strict";

const child_process = require("child_process");
const Z = require("./geometry/glyph-point");
const JSONStream = require("JSONStream");
const which = require("which");
const path = require("path");
const fs = require("fs-extra");

function getStream(sourcefile, options) {
	const ext = path.parse(sourcefile).ext;
	if (ext === ".ttf" || ext === ".otf") {
		const cp = child_process.spawn(which.sync(`otfccdump`), [
			sourcefile,
			"--glyph-name-prefix",
			options.partname + "/",
			"--ignore-hints",
			"--no-bom",
			"--decimal-cmap"
		]);
		return cp.stdout;
	} else {
		return fs.createReadStream(sourcefile);
	}
}

function introduce(ctx, partname, sourcefile, options) {
	const t = this;
	return new Promise(function(resolve, reject) {
		let font = {};
		getStream(sourcefile, { partname: partname })
			.pipe(JSONStream.parse("$*"))
			.on("data", function(data) {
				font[data.key] = data.value;
			})
			.on("close", function() {
				for (let k in font.glyf) {
					let glyph = font.glyf[k];
					if (!glyph.contours) continue;
					for (let contour of glyph.contours) {
						for (let j = 0; j < contour.length; j++) {
							contour[j] = new Z(contour[j].x, contour[j].y, contour[j].on);
						}
					}
				}
				t.introduce(partname, font);
				return resolve(font);
			});
	});<commit_msg>set explicit encoding for buffers<commit_after>"use strict";

const child_process = require("child_process");
const Z = require("./geometry/glyph-point");
const JSONStream = require("JSONStream");
const which = require("which");
const path = require("path");
const fs = require("fs-extra");

function getStream(sourcefile, options) {
	const ext = path.parse(sourcefile).ext;
	if (ext === ".ttf" || ext === ".otf") {
		const cp = child_process.spawn(which.sync(`otfccdump`), [
			sourcefile,
			"--glyph-name-prefix",
			options.partname + "/",
			"--ignore-hints",
			"--no-bom",
			"--decimal-cmap"
		]);
		cp.stdout.setEncoding("utf8");
		return cp.stdout;
	} else {
		return fs.createReadStream(sourcefile, { encoding: "utf8" });
	}
}

function introduce(ctx, partname, sourcefile, options) {
	const t = this;
	return new Promise(function(resolve, reject) {
		let font = {};
		getStream(sourcefile, { partname: partname })
			.pipe(JSONStream.parse("$*"))
			.on("data", function(data) {
				font[data.key] = data.value;
			})
			.on("close", function() {
				for (let k in font.glyf) {
					let glyph = font.glyf[k];
					if (!glyph.contours) continue;
					for (let contour of glyph.contours) {
						for (let j = 0; j < contour.length; j++) {
							contour[j] = new Z(contour[j].x, contour[j].y, contour[j].on);
						}
					}
				}
				t.introduce(partname, font);
				return resolve(font);
			});
	});
{'fraction_non_alphanumeric': 0.18469945355191256, 'fraction_numerical': 0.0014571948998178506, 'mean_word_length': 3.457792207792208, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6456844', 'n_tokens_mistral': 1124, 'n_tokens_neox': 942, 'n_words': 209}