<commit_before>"use strict";

var ensurePropertyExists = function(d,i) {
    if (!d[i]) d[i] = {};
    return d[i];
}

var createData = function(pc, tokens) {
    var data = {},
        n = tokens.length;

    for (var i=0;i<n-(pc-1);++i) {
	var obj = ensurePropertyExists(data,tokens[i]);

	for (var j=1;j<pc-1;++j) {
	    obj[tokens[i+j]] = {};
	    obj = obj[tokens[i+j]];
	}

	if (!obj[tokens[i+pc-1]])
	    obj[tokens[i+pc-1]] = 0;

	obj[tokens[i+pc-1]]++;
    }

    return data;
};

var Model = function(predictionContext, tokens) {

    var n = tokens.length;

    this._data = createData(predictionContext, tokens);

    this.entryCount = function() {
	return tokens.length;
    };

    this.frequencyOf = function() {
	var end = this._data;

	for (var i=0;i<arguments.length;++i) {
	    end = end[arguments[i]];
	}

	return end;
    };

    this.pickOne = function(soFar, upTo) {
	var n = soFar.length;

	// TODO this needs to reach the end of the prediction
	// chain.
	var startIndex = n - (predictionContext - 1);
	var obj = this._data;
	for (var i = startIndex; i < upTo; i++)
	    obj = this._data[soFar[i]];

	return this.weightedChoice(obj);
    };

    this.weightedChoice = function(obj) {

	var words = [];

	for(var prop in obj) {
	    for (var i=0;i<obj[prop];++i) {
		words.push(prop);
	    }
	}
	
	var n = Math.random() * words.length | 0;
	return words[n];
    };

    this.generate = function(start, len) {
	var r = [];
	r[0] = tokens[start];

	for (var i=1;i<len;++i) {
	    r[i] = this.pickOne(r,i);
	}

	return r;
    };

    return this;
};

var buildModel = function(predictionContext, source) { 
    return new Model(predictionContext, source); 
};

exports.buildModel = buildModel;
<commit_msg>use the start phrase to prime<commit_after>"use strict";

var ensurePropertyExists = function(d,i) {
    if (!d[i]) d[i] = {};
    return d[i];
}

var createData = function(pc, tokens) {
    var data = {},
        n = tokens.length;

    for (var i=0;i<n-(pc-1);++i) {
	var obj = ensurePropertyExists(data,tokens[i]);

	for (var j=1;j<pc-1;++j) {
	    obj[tokens[i+j]] = {};
	    obj = obj[tokens[i+j]];
	}

	if (!obj[tokens[i+pc-1]])
	    obj[tokens[i+pc-1]] = 0;

	obj[tokens[i+pc-1]]++;
    }

    return data;
};

var Model = function(predictionContext, tokens) {

    var n = tokens.length;

    this._data = createData(predictionContext, tokens);

    this.entryCount = function() {
	return tokens.length;
    };

    this.frequencyOf = function() {
	var end = this._data;

	for (var i=0;i<arguments.length;++i) {
	    end = end[arguments[i]];
	}

	return end;
    };

    this.pickOne = function(soFar, upTo) {
	var n = soFar.length;

	// TODO this needs to reach the end of the prediction
	// chain.
	var startIndex = n - (predictionContext - 1);
	var obj = this._data;
	for (var i = startIndex; i < upTo; i++)
	    obj = this._data[soFar[i]];

	return this.weightedChoice(obj);
    };

    this.weightedChoice = function(obj) {

	var words = [];

	for(var prop in obj) {
	    for (var i=0;i<obj[prop];++i) {
		words.push(prop);
	    }
	}
	
	var n = Math.random() * words.length | 0;
	return words[n];
    };

    this.generate = function(startPhrase, len) {
	var r = startPhrase;

	for (var i=1;i<len;++i) {
	    r[i] = this.pickOne(r,i);
	}

	return r;
    };

    return this;
};

var buildModel = function(predictionContext, source) { 
    return new Model(predictionContext, source); 
};

exports.buildModel = buildModel;

{'fraction_non_alphanumeric': 0.17446562680531486, 'fraction_numerical': 0.00779896013864818, 'mean_word_length': 2.9531963470319633, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1949841', 'n_tokens_mistral': 1382, 'n_tokens_neox': 1293, 'n_words': 292}