<commit_before>var $ = require('jquery');
//this is a mapping from the class names (generic ones, for compatability with codemirror themes), to what they -actually- represent
var tokenTypes = {
	"string-2" : "prefixed",
	"atom": "var"
};

module.exports = function(yasqe, completerName) {
	//this autocompleter also fires on-change!
	yasqe.on("change", function() {
		module.exports.appendPrefixIfNeeded(yasqe, completerName);
	});
	
	
	return {
		isValidCompletionPosition : function(){module.exports.isValidCompletionPosition(yasqe);},
		get : function(token, callback) {
			$.get("http://prefix.cc/popular/all.file.json", function(data) {
				var prefixArray = [];
				for ( var prefix in data) {
					if (prefix == "bif")
						continue;// skip this one! see #231
					var completeString = prefix + ": <" + data[prefix] + ">";
					prefixArray.push(completeString);// the array we want to store in localstorage
				}
				
				prefixArray.sort();
				callback(prefixArray);
			});
		},
		preProcessToken: function(token) {module.exports.preprocessPrefixTokenForCompletion(yasqe, token)},
		async : true,
		bulk : true,
		autoShow : true,
		persistent : completerName,
	};
};
module.exports.isValidCompletionPosition = function(yasqe) {
	var cur = yasqe.getCursor(), token = yasqe.getTokenAt(cur);

	// not at end of line
	if (yasqe.getLine(cur.line).length > cur.ch)
		return false;

	if (token.type != "ws") {
		// we want to complete token, e.g. when the prefix starts with an a
		// (treated as a token in itself..)
		// but we to avoid including the PREFIX tag. So when we have just
		// typed a space after the prefix tag, don't get the complete token
		token = yasqe.getCompleteToken();
	}

	// we shouldnt be at the uri part the prefix declaration
	// also check whether current token isnt 'a' (that makes codemirror
	// thing a namespace is a possiblecurrent
	if (!token.string.indexOf("a") == 0<commit_msg>removed unused option<commit_after>var $ = require('jquery');
//this is a mapping from the class names (generic ones, for compatability with codemirror themes), to what they -actually- represent
var tokenTypes = {
	"string-2" : "prefixed",
	"atom": "var"
};

module.exports = function(yasqe, completerName) {
	//this autocompleter also fires on-change!
	yasqe.on("change", function() {
		module.exports.appendPrefixIfNeeded(yasqe, completerName);
	});
	
	
	return {
		isValidCompletionPosition : function(){module.exports.isValidCompletionPosition(yasqe);},
		get : function(token, callback) {
			$.get("http://prefix.cc/popular/all.file.json", function(data) {
				var prefixArray = [];
				for ( var prefix in data) {
					if (prefix == "bif")
						continue;// skip this one! see #231
					var completeString = prefix + ": <" + data[prefix] + ">";
					prefixArray.push(completeString);// the array we want to store in localstorage
				}
				
				prefixArray.sort();
				callback(prefixArray);
			});
		},
		preProcessToken: function(token) {module.exports.preprocessPrefixTokenForCompletion(yasqe, token)},
		async : true,
		bulk : true,
		persistent : completerName,
	};
};
module.exports.isValidCompletionPosition = function(yasqe) {
	var cur = yasqe.getCursor(), token = yasqe.getTokenAt(cur);

	// not at end of line
	if (yasqe.getLine(cur.line).length > cur.ch)
		return false;

	if (token.type != "ws") {
		// we want to complete token, e.g. when the prefix starts with an a
		// (treated as a token in itself..)
		// but we to avoid including the PREFIX tag. So when we have just
		// typed a space after the prefix tag, don't get the complete token
		token = yasqe.getCompleteToken();
	}

	// we shouldnt be at the uri part the prefix declaration
	// also check whether current token isnt 'a' (that makes codemirror
	// thing a namespace is a possiblecurrent
	if (!token.string.indexOf("a") == 0
{'fraction_non_alphanumeric': 0.1291503267973856, 'fraction_numerical': 0.00261437908496732, 'mean_word_length': 3.7002457002457003, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2465601', 'n_tokens_mistral': 1343, 'n_tokens_neox': 1212, 'n_words': 371}