<commit_before>const exec = require('child_process').spawn;
const fs = require('fs');

const csv = require('csv-parser');
var lineReader = require('line-reader');

const merging = require('./fasta-merging.js');
const toolbox = require('../toolbox.js');
const otu_manager = require('../otu_manager.js');


exports.name = 'otu-vsearch';
exports.multicore = true;
exports.category = 'Clustering';

exports.run = function (os, config, callback) {
	let token = os.token;
	let directory = '/app/data/' + token + '/';
	let tmp_merged = toolbox.tmp_filename() + '.fasta';
	let tmp_origins = toolbox.tmp_filename() + '.tsv';

	let otu_file = config.params.outputs.otus_table;

	let config_merging = {params:{
		inputs: config.params.inputs,
		outputs: {
			merged: tmp_merged,
			origins: tmp_origins
		},
		params: {}
	}, log:config.log};

	// Merging
	merging.run (os, config_merging, (_, msg) => {
		if (msg != null) {
			callback(os, msg);
			return;
		}

		// OTU search
		config.params.inputs.merged = tmp_merged;
		config.params.inputs.origins = tmp_origins;
		otu_search(os, config, (os, msg) => {
			// Remove unnecessary files
			fs.unlink(directory+tmp_origins, ()=>{});
			fs.unlink(directory+tmp_merged, ()=>{});

			callback(os, msg);
		});
	});
};

var otu_search = (os, config, callback) => {
	let directory = '/app/data/' + os.token + '/';
	// let tmp_output = toolbox.tmp_filename() + '.txt';
	let tmp_output = toolbox.tmp_filename() + '.uc';
	let in_reads = config.params.inputs.merged;
	let centroids_file = config.params.outputs.centroids;
	// let tmp_centroids = toolbox.tmp_filename() + '.fasta';

	// config.params.inputs.tmp_centroids = tmp_centroids;

	// Clustering options
	var options = ['--cluster_fast', directory + in_reads,
		'--sizein', '--sizeout',
		'--uc', directory + tmp_output,
		'--id', config.params.params.similarity,
		'--centroids', directory + centroids_file,
		'--threads', os.cores,
		'--qmask', 'none', '--minseqlength', '1'];

	// Execute vsearch
	console.log("Running vsearch with the command line:");
	console.log('/app/lib/vsearch/bin/vsearch', options.join(' '));
	var child = exec('/app/lib/vsearch/bin/vsearch', options);

	child.stdout.on('data', function(data) {
		fs.appendFileSync(directory + config.log, data);
	});
	child.stderr.on('data', function(data) {
		fs.appendFileSync(directory + config.log, data);
	});
	child.on('close', function(code) {
		if (code != 0)
			callback();
		else {
			config.params.inputs.uc = tmp_output;
			config.params.params.ordered = config.params.params.ordered_vsearch;<commit_msg>otu-vsearch presort by abundance instead of by length<commit_after>const exec = require('child_process').spawn;
const fs = require('fs');

const csv = require('csv-parser');
var lineReader = require('line-reader');

const merging = require('./fasta-merging.js');
const toolbox = require('../toolbox.js');
const otu_manager = require('../otu_manager.js');


exports.name = 'otu-vsearch';
exports.multicore = true;
exports.category = 'Clustering';

exports.run = function (os, config, callback) {
	let token = os.token;
	let directory = '/app/data/' + token + '/';
	let tmp_merged = toolbox.tmp_filename() + '.fasta';
	let tmp_origins = toolbox.tmp_filename() + '.tsv';

	let otu_file = config.params.outputs.otus_table;

	let config_merging = {params:{
		inputs: config.params.inputs,
		outputs: {
			merged: tmp_merged,
			origins: tmp_origins
		},
		params: {}
	}, log:config.log};

	// Merging
	merging.run (os, config_merging, (_, msg) => {
		if (msg != null) {
			callback(os, msg);
			return;
		}

		// OTU search
		config.params.inputs.merged = tmp_merged;
		config.params.inputs.origins = tmp_origins;
		otu_search(os, config, (os, msg) => {
			// Remove unnecessary files
			fs.unlink(directory+tmp_origins, ()=>{});
			fs.unlink(directory+tmp_merged, ()=>{});

			callback(os, msg);
		});
	});
};

var otu_search = (os, config, callback) => {
	let directory = '/app/data/' + os.token + '/';
	// let tmp_output = toolbox.tmp_filename() + '.txt';
	let tmp_output = toolbox.tmp_filename() + '.uc';
	let in_reads = config.params.inputs.merged;
	let centroids_file = config.params.outputs.centroids;
	// let tmp_centroids = toolbox.tmp_filename() + '.fasta';

	// config.params.inputs.tmp_centroids = tmp_centroids;

	// Clustering options
	var options = ['--cluster_size', directory + in_reads,
		'--sizein', '--sizeout',
		'--uc', directory + tmp_output,
		'--id', config.params.params.similarity,
		'--centroids', directory + centroids_file,
		'--threads', os.cores,
		'--qmask', 'none', '--minseqlength', '1'];

	// Execute vsearch
	console.log("Running vsearch with the command line:");
	console.log('/app/lib/vsearch/bin/vsearch', options.join(' '));
	var child = exec('/app/lib/vsearch/bin/vsearch', options);

	child.stdout.on('data', function(data) {
		fs.appendFileSync(directory + config.log, data);
	});
	child.stderr.on('data', function(data) {
		fs.appendFileSync(directory + config.log, data);
	});
	child.on('close', function(code) {
		if (code != 0)
			callback();
		else {
			config.params.inputs.uc = tmp_output;
			config.params.params.ordered = config.params.params.ordered_vsearch;
{'fraction_non_alphanumeric': 0.18071124855044454, 'fraction_numerical': 0.0007730962504831851, 'mean_word_length': 4.029154518950437, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6933929', 'n_tokens_mistral': 1943, 'n_tokens_neox': 1807, 'n_words': 360}