<commit_before>﻿var through = require('through');
var traverse = require('traverse');
var path = require('path');

function isJSON(file) {
	return (/\.json$/).test(file);
}

function pointify(file, opts) {

	var wd = path.dirname(file) + '/';
	var data = '';

	opts = opts || {};

	var marshall = function(data, cwd, cb) {

		traverse(data).
		forEach(function(value) {

			if (this.key === '$ref') {
				var mod = value;
				var update;
				var paths;
				var foundInPaths = false;
				var contents;
				var node = this.parent.node;
				var keys = Object.keys(node);

				try {
					contents = require(mod);
				} catch (e0) {
					try {
						mod = cwd + value;
						contents = require(mod);
					} catch (e1) {

						if (!opts.paths) throw e1;

						paths = (Array.isArray(opts.paths)) ?
							opts.paths : [opts.paths];

						paths.forEach(function(path) {

							if (!foundInPaths) {
								try {

									mod = path + '/' + value;
									contents = require(mod);
									foundInPaths = true;

								} catch (e2) {

								}
							}

						});

						if (!foundInPaths) throw e1;

					}


				}

				update = JSON.parse(JSON.stringify(contents));

				if (keys.length > 1)
					keys.forEach(function(key) {
						if (key !== '$ref')
							update[key] = node[key];
					});

				marshall(update, path.dirname(mod) + '/', function(data) {
					this.parent.update(data);
				}.bind(this));
			}
		});

		return cb(data);

	};

	var write = function(buf) {
		data += buf;
	};

	var end = function() {

		marshall(JSON.parse(data), wd, function(data) {
			this.queue(JSON.stringify(data));
			this.queue(null);
		}.bind(this));


	};

	if (!isJSON(file)) return through();
	return through(write, end);
}

module.exports = pointify;
<commit_msg>More useful error message when json is invalid<commit_after>﻿var through = require('through');
var traverse = require('traverse');
var path = require('path');

function isJSON(file) {
	return (/\.json$/).test(file);
}

function pointify(file, opts) {

	var wd = path.dirname(file) + '/';
	var data = '';

	opts = opts || {};

	var marshall = function(data, cwd, cb) {

		traverse(data).
		forEach(function(value) {

			if (this.key === '$ref') {
				var mod = value;
				var update;
				var paths;
				var foundInPaths = false;
				var contents;
				var node = this.parent.node;
				var keys = Object.keys(node);

				try {
					contents = require(mod);
				} catch (e0) {
					try {
						mod = cwd + value;
						contents = require(mod);
					} catch (e1) {

						if (!opts.paths) throw e1;

						paths = (Array.isArray(opts.paths)) ?
							opts.paths : [opts.paths];

						paths.forEach(function(path) {

							if (!foundInPaths) {
								try {

									mod = path + '/' + value;
									contents = require(mod);
									foundInPaths = true;

								} catch (e2) {

								}
							}

						});

						if (!foundInPaths) throw e1;

					}


				}

				update = JSON.parse(JSON.stringify(contents));

				if (keys.length > 1)
					keys.forEach(function(key) {
						if (key !== '$ref')
							update[key] = node[key];
					});

				marshall(update, path.dirname(mod) + '/', function(data) {
					this.parent.update(data);
				}.bind(this));
			}
		});

		return cb(data);

	};

	var write = function(buf) {
		data += buf;
	};

	var end = function() {

          var obj;

          try {
             obj = JSON.parse(data);
          }catch(e) {
            throw new Error('Uncaught error while procesing '+file+': '+e.message);
          }

		marshall(obj, wd, function(data) {
			this.queue(JSON.stringify(data));
			this.queue(null);
		}.bind(this));


	};

	if (!isJSON(file)) return through();
	return through(write, end);
}

module.exports = pointify;

{'fraction_non_alphanumeric': 0.16684520621317622, 'fraction_numerical': 0.0032137118371719335, 'mean_word_length': 2.2591623036649215, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1125655', 'n_tokens_mistral': 1690, 'n_tokens_neox': 1255, 'n_words': 281}