<commit_before>// 
// Copyright (c) Microsoft and contributors.  All rights reserved.
// 
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//   http://www.apache.org/licenses/LICENSE-2.0
// 
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// 
// See the License for the specific language governing permissions and
// limitations under the License.
// 

import ctxm = require('./context');

//
// Utilities passed to each task
// which provides contextual logging to server etc...
//
export class Utilities {
	constructor(context: ctxm.Context) {
		this.ctx = context;
	}

	private ctx: ctxm.Context;

    // spawn a process with stdout/err piped to context's logger
	// callback(err)
	public spawn(name: string, args: string[], options, callback) {
		options = options || {};

		var ops = {
			cwd: process.cwd(),
			env: process.env,
			failOnStdErr: true 
		};

		// write over specified options over default options (ops)
		for (var op in options) {
			ops[op] = options[op];
		}

        this.ctx.info('cwd: ' + ops.cwd);
        this.ctx.info(name + ' ' + args);

		var cp = require('child_process').spawn;

		var runCP = cp(name, args, ops);

		runCP.stdout.on('data', (data) => { 			
		  	this.ctx.info(data.toString('utf8'));
		});

		runCP.stderr.on('data', (data) => {
			if (ops.failOnStdErr) {
				callback(new Error(data.toString('utf8')));
			}
			else {
				this.ctx.info(data.toString('utf8'));
			}
		});

		runCP.on('exit', (code) => {
			if (code == 0) {
				callback();
			}
			else {
				var msg = 'Return code: ' + code;
				this.ctx.error(msg);
				callback(new Error(msg));
			}
		});			
	}	
}<commit_msg>shell script should wait for exit even on failure<commit_after>// 
// Copyright (c) Microsoft and contributors.  All rights reserved.
// 
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//   http://www.apache.org/licenses/LICENSE-2.0
// 
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// 
// See the License for the specific language governing permissions and
// limitations under the License.
// 

import ctxm = require('./context');

//
// Utilities passed to each task
// which provides contextual logging to server etc...
//
export class Utilities {
	constructor(context: ctxm.Context) {
		this.ctx = context;
	}

	private ctx: ctxm.Context;

    // spawn a process with stdout/err piped to context's logger
	// callback(err)
	public spawn(name: string, args: string[], options, callback) {
		var failed = false;
		options = options || {};

		var ops = {
			cwd: process.cwd(),
			env: process.env,
			failOnStdErr: true 
		};

		// write over specified options over default options (ops)
		for (var op in options) {
			ops[op] = options[op];
		}

        this.ctx.info('cwd: ' + ops.cwd);
        this.ctx.info(name + ' ' + args);

		var cp = require('child_process').spawn;

		var runCP = cp(name, args, ops);

		runCP.stdout.on('data', (data) => { 			
		  	this.ctx.info(data.toString('utf8'));
		});

		runCP.stderr.on('data', (data) => {
			failed = ops.failOnStdErr;
            this.ctx.error(data.toString('utf8'));
		});

		runCP.on('exit', (code) => {
            if (failed) {
                callback(new Error('Failed with Error Output'));
                return;
            }

            if (code == 0) {
                callback();
            } else {                
                var msg = 'Return code: ' + code;
                this.ctx.error(msg);

                callback(new Error(msg));
            }
		});			
	}	
}
{'fraction_non_alphanumeric': 0.13896390866683034, 'fraction_numerical': 0.003682789098944267, 'mean_word_length': 2.5272727272727273, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4702306', 'n_tokens_mistral': 1354, 'n_tokens_neox': 1219, 'n_words': 400}