<commit_before>const lineReader = require('line-reader');
const fs = require('fs');



exports.write_otu_table = (otu_matrix, table_file, t2sOrder, callback) => {
	console.log('Write OTU matrix');

	// --- file order from matrix ---
	// Get the experiments objects
	let fileOrder = Object.values(otu_matrix);
	// Transform arrays into arguments for Object.assign
	fileOrder = [{}].concat(fileOrder);
	// Get unique keys by merging the objects
	fileOrder = Object.keys(Object.assign.apply(this, fileOrder));

	// Final order
	let order = [];
	let header = [];
	if (t2sOrder == null) {
		order = fileOrder;
		header = fileOrder;
	} else {
		for (let o_idx=0 ; o_idx<t2sOrder.length ; o_idx++) {
			let t2s_name = t2sOrder[o_idx];

			for (let f_idx=0 ; f_idx<fileOrder.length ; f_idx++) {
				let filename = fileOrder[f_idx];

				if (filename.includes(t2s_name)) {
					order.push(filename);
					header.push(t2s_name);
				}
			}
		}
	}


	// --- Write OTU table to file ---
	// Header
	fs.closeSync(fs.openSync(table_file, 'w'));
	fs.appendFileSync(table_file, 'OTU');
	for (let exp_idx=0 ; exp_idx<header.length ; exp_idx++)
		fs.appendFileSync(table_file, '\t' + header[exp_idx]);
	fs.appendFileSync(table_file, '\n');

	// Content
	for (let cluster_idx in otu_matrix) {
		fs.appendFileSync(table_file, cluster_idx);

		for (let exp_idx=0 ; exp_idx<order.length ; exp_idx++) {
			let exp = order[exp_idx];

			// Write the value if present, else 0
			if (otu_matrix[cluster_idx][exp]) {
				fs.appendFileSync(table_file, '\t' + otu_matrix[cluster_idx][exp]);
			} else {
				fs.appendFileSync(table_file, '\t0');
			}
		}
		fs.appendFileSync(table_file, '\n');
	}

	callback();
};


exports.load_origins_matrix = (origins_file, callback) => {
	let origins = {experiments:[]};

	lineReader.eachLine(origins_file, function(line, last) {
		line = line.split('\t');

		origins[line[0]] = {};
		for (let exIdx=1 ; exIdx<line.length ; exIdx++) {
			let origin = line[exIdx];
			let size = origin.substr(origin.indexOf(';size=')+6);
			size = parseInt(size.substr(0, size.indexOf(';')));
			origin = origin.substr(0, origin.indexOf(';size='));

			// Add the origin
			origins[line[0]][origin] = size;

			// Save the experiment
			if (!origins.experiments.includes(origin))
				origins.experiments.push(origin);
		}

		if (last)
			callback(origins);
	});
};

<commit_msg>optimization of file writing<commit_after>const lineReader = require('line-reader');
const fs = require('fs');



exports.write_otu_table = (otu_matrix, table_file, t2sOrder, callback) => {
	console.log('Write OTU matrix');

	// --- file order from matrix ---
	// Get the experiments objects
	let fileOrder = Object.values(otu_matrix);
	// Transform arrays into arguments for Object.assign
	fileOrder = [{}].concat(fileOrder);
	// Get unique keys by merging the objects
	fileOrder = Object.keys(Object.assign.apply(this, fileOrder));

	// Final order
	let order = [];
	let header = [];
	if (t2sOrder == null) {
		order = fileOrder;
		header = fileOrder;
	} else {
		for (let o_idx=0 ; o_idx<t2sOrder.length ; o_idx++) {
			let t2s_name = t2sOrder[o_idx];

			for (let f_idx=0 ; f_idx<fileOrder.length ; f_idx++) {
				let filename = fileOrder[f_idx];

				if (filename.includes(t2s_name)) {
					order.push(filename);
					header.push(t2s_name);
				}
			}
		}
	}


	// --- Write OTU table to file ---
	let stream = fs.createWriteStream(table_file, {'flags': 'a'});
	// Header
	fs.closeSync(fs.openSync(table_file, 'w'));
	stream.write('OTU');
	for (let exp_idx=0 ; exp_idx<header.length ; exp_idx++)
		stream.write('\t' + header[exp_idx]);
	stream.write('\n');

	// Content
	for (let cluster_idx in otu_matrix) {
		stream.write(cluster_idx);

		for (let exp_idx=0 ; exp_idx<order.length ; exp_idx++) {
			let exp = order[exp_idx];

			// Write the value if present, else 0
			if (otu_matrix[cluster_idx][exp]) {
				stream.write('\t' + otu_matrix[cluster_idx][exp]);
			} else {
				stream.write('\t0');
			}
		}
		stream.write('\n');
	}

	callback();
};


exports.load_origins_matrix = (origins_file, callback) => {
	let origins = {experiments:[]};

	lineReader.eachLine(origins_file, function(line, last) {
		line = line.split('\t');

		origins[line[0]] = {};
		for (let exIdx=1 ; exIdx<line.length ; exIdx++) {
			let origin = line[exIdx];
			let size = origin.substr(origin.indexOf(';size=')+6);
			size = parseInt(size.substr(0, size.indexOf(';')));
			origin = origin.substr(0, origin.indexOf(';size='));

			// Add the origin
			origins[line[0]][origin] = size;

			// Save the experiment
			if (!origins.experiments.includes(origin))
				origins.experiments.push(origin);
		}

		if (last)
			callback(origins);
	});
};


{'fraction_non_alphanumeric': 0.1685823754789272, 'fraction_numerical': 0.008088548318433376, 'mean_word_length': 3.7226130653266334, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4700605', 'n_tokens_mistral': 1875, 'n_tokens_neox': 1688, 'n_words': 349}