<commit_before>var docs = null;
var tags = {};
var dates = {
	from : null,
	fromEnabled : false,
	to : null,
	toEnabled : false,
	newestFirst : false
};
var docIdsDisplayed = [];

var renderDoc = function(doc, cache) {
	if (cache[doc.id] == null) {
		var render = {
			clickListener : function(id){
				return function(event) {
					if (!$(event.target).is("a")) {
						openDoc(id);
					}
				}
			}(doc.id),
			html : ""
		};
		render.html = '<div class="doc">';
		render.html +=  '<div class="thumb">';
		render.html +=    doc.thumb ? '<img src="/fetch?id='+doc.id+'&type=thumb"/>' : 'No thumb';
		render.html +=    '<br><a href="/fetch?id='+doc.id+'&type=raw" target="_blank">Download</a>';
		render.html +=  '</div>';
		render.html +=  '<div><span class="key">Filename</span><span class="value">'+doc.filename+'</span></div>';
		render.html +=  '<div><span class="key">Uploaded</span><span class="value">'+doc.uploaded+'</span></div>';
		render.html +=  '<div><span class="key">Effective</span><span class="value">'+doc.effective+'</span></div>';
		render.html +=  '<ul>';

		for (var i = 0; i < doc.tags.length; i++) {
			render.html += '<li>'+doc.tags[i]+'</li>';
		}
		render.html += '</ul>';
		render.html += '</div>';

		cache[doc.id] = render;
	}
	return cache[doc.id];
};

var redraw = function() {
	var tagFilter = $('#tag-filter');
	tagFilter.empty();
	var tagNames = Object.keys(tags);
	tagNames.sort();
	for (var i = 0; i < tagNames.length; i++) {
		var element = $('<span class="tag'+( (tags[tagNames[i]]) ? ' selected' : '' )+'">'+tagNames[i]+'</span>');
		element.click(function(event) {
			var tmp = $(this);
			var tagName = tmp.html();
			if (tmp.hasClass('selected')) {
				tmp.removeClass('selected');
				tags[tagName] = false;
			} else {
				tmp.addClass('selected');
				tags[tagName] = true;
			}
			redraw();
		});
		tagFilter.append(element);
	}
	
	// Filter docs for display
	var allFiles = [];
	var newFiles = [];
	for (var id in docs) {
		var doc = docs[id];

		// Filter out by tags
		var showInAll = false;
		for (var i = 0; i < doc.tags.length; i++) {
			if (tags[doc.tags[i]]) {
				showInAll = true;
				break;
			}
		}
		if (doc.tags.length == 0) {
			// Special case for untagged docs
			showInAll = true;
			for (var tagName in tags) {
				if (tags[tagName]) {
					showInAll = false;
					break;
				}
			}
		}
		
		// Filter by date
		if (showInAll && dates.fromEnabled) {
			if (doc.effective < dates.from) {
				showInAll = false;
			}
		}
		if (showInAll && dates.toEnabled) {
			if (dates.to < doc.effective) {
				showInAll = false;
			}
		}

		if (showInAll) {
			allFiles.push(doc);
		}

		if (doc.unseen) {
			newFiles.push(doc);
		}
	}
<commit_msg>Unseen documents should not show up in All Files<commit_after>var docs = null;
var tags = {};
var dates = {
	from : null,
	fromEnabled : false,
	to : null,
	toEnabled : false,
	newestFirst : false
};
var docIdsDisplayed = [];

var renderDoc = function(doc, cache) {
	if (cache[doc.id] == null) {
		var render = {
			clickListener : function(id){
				return function(event) {
					if (!$(event.target).is("a")) {
						openDoc(id);
					}
				}
			}(doc.id),
			html : ""
		};
		render.html = '<div class="doc">';
		render.html +=  '<div class="thumb">';
		render.html +=    doc.thumb ? '<img src="/fetch?id='+doc.id+'&type=thumb"/>' : 'No thumb';
		render.html +=    '<br><a href="/fetch?id='+doc.id+'&type=raw" target="_blank">Download</a>';
		render.html +=  '</div>';
		render.html +=  '<div><span class="key">Filename</span><span class="value">'+doc.filename+'</span></div>';
		render.html +=  '<div><span class="key">Uploaded</span><span class="value">'+doc.uploaded+'</span></div>';
		render.html +=  '<div><span class="key">Effective</span><span class="value">'+doc.effective+'</span></div>';
		render.html +=  '<ul>';

		for (var i = 0; i < doc.tags.length; i++) {
			render.html += '<li>'+doc.tags[i]+'</li>';
		}
		render.html += '</ul>';
		render.html += '</div>';

		cache[doc.id] = render;
	}
	return cache[doc.id];
};

var redraw = function() {
	var tagFilter = $('#tag-filter');
	tagFilter.empty();
	var tagNames = Object.keys(tags);
	tagNames.sort();
	for (var i = 0; i < tagNames.length; i++) {
		var element = $('<span class="tag'+( (tags[tagNames[i]]) ? ' selected' : '' )+'">'+tagNames[i]+'</span>');
		element.click(function(event) {
			var tmp = $(this);
			var tagName = tmp.html();
			if (tmp.hasClass('selected')) {
				tmp.removeClass('selected');
				tags[tagName] = false;
			} else {
				tmp.addClass('selected');
				tags[tagName] = true;
			}
			redraw();
		});
		tagFilter.append(element);
	}
	
	// Filter docs for display
	var allFiles = [];
	var newFiles = [];
	for (var id in docs) {
		var doc = docs[id];

		// Filter out by tags
		var showInAll = false;
		if (!doc.unseen) {
			for (var i = 0; i < doc.tags.length; i++) {
				if (tags[doc.tags[i]]) {
					showInAll = true;
					break;
				}
			}
			if (doc.tags.length == 0) {
				// Special case for untagged docs
				showInAll = true;
				for (var tagName in tags) {
					if (tags[tagName]) {
						showInAll = false;
						break;
					}
				}
			}
		}
		
		// Filter by date
		if (showInAll && dates.fromEnabled) {
			if (doc.effective < dates.from) {
				showInAll = false;
			}
		}
		if (showInAll && dates.toEnabled) {
			if (dates.to < doc.effective) {
				showInAll = false;
			}
		}

		if (showInAll) {
			allFiles.push(doc);
		}

		if (doc.unseen) {
			newFiles.push(doc);
		}
	}

{'fraction_non_alphanumeric': 0.21277372262773722, 'fraction_numerical': 0.00145985401459854, 'mean_word_length': 3.0539940828402368, 'pattern_counts': {'":': 0, '<': 77, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4415999', 'n_tokens_mistral': 2337, 'n_tokens_neox': 2034, 'n_words': 428}