<commit_before>"use strict"

var API = "_data"
var GLOBAL = {
	items: {},
	sources: {},
	xhrs: {},
}

function set_image_class() {
	var img = $(this)
	if (img.attr("class") == "article-image") {
		if (img.width() > 2 * img.height()) {
			img.attr("class", "article-image-w")
		} else {
			img.attr("class", "article-image-h")
		}
	}
}

function make_item_box(item, source) {
	var image = $("<img>").attr("src", item.image ? API + "/" + item.image + ".png" : "icons/" + source.id + ".png")
		                  .attr("alt", item.image ? item.title : source.name)
		                  .attr("class", item.image ? "article-image" : "article-image-dummy")
		                  .on("load", set_image_class)
    var source_image = null
    if (item.image) {
    	var source_image = $("<img>").attr("src", "icons/" + source.id + ".png")
    	                             .attr("alt", source.name)
    	                             .attr("class", "source-image")
    }
	var imagebox = $("<div>").attr("class", "image")
	                         .append(image)
	if (source_image) {
		imagebox.append(source_image)
	}

	var headera = $("<a>").text(item.title)
	                      .attr("href", item.url)
	var header = $("<h2>").append(headera)
	var summary = $("<p>").text(item.summary)
	var content = $("<div>").attr("class", "content")
	                        .append(header)
	                        .append(summary)

	var article = $("<article>").attr("lang", source.lang)
	                            .attr("id", "article-" + item.id)
	                            .append(imagebox)
	                            .append(content)
	return article
}

function refresh_news_display() {
	var keys = []
	for (var key in GLOBAL.items) {
		keys.push(key)
	}
	keys.sort(function(a, b) {
		var akey = GLOBAL.sources[a].name.toLowerCase()
		var bkey = GLOBAL.sources[b].name.toLowerCase()
		return (akey < bkey ? -1 : (akey > bkey ? 1 : 0))
	})

	var i = 0
	var insert_anchor = $("#anchor")
	$("#items article").attr("data-delete", "data-delete")
	do {
		var has_hit = false
		var retry = 0
		for (var k = 0; k < keys.length; ++k) {
			var item = GLOBAL.items[keys[k]][i + retry]
			if (item) {
				has_hit = true

				var box = $("#article-" + item.id)
				if (box.length && !box.attr("data-delete")) {
					// article already in DOM because item is a duplicate
					// retry with next article
					--k
					++retry
					continue
				} else if (box.length) {
					// article already in DOM
					box.attr("data-delete", null)
				} else {
					// article not added yet
					box = make_item_box(item, GLOBAL.sources[keys[k]])
				}
				insert_anchor.before(box)
				retry = 0
			}
		}
		++i
	} while (has_hit)

	$("#items article[data-delete]").remove()

	$("#lastupdate").text(new Date().toLocaleString(
		window.navigator.userLanguage || window.navigator.language, {
		weekday: "long",
		year: "numeric",
		month: "long",
		day: "numeric",
		hour: "numeric",
		minute: "numeric",
	}))
}
<commit_msg>fix but with retry in client with double elements at the end of the item list<commit_after>"use strict"

var API = "_data"
var GLOBAL = {
	items: {},
	sources: {},
	xhrs: {},
}

function set_image_class() {
	var img = $(this)
	if (img.attr("class") == "article-image") {
		if (img.width() > 2 * img.height()) {
			img.attr("class", "article-image-w")
		} else {
			img.attr("class", "article-image-h")
		}
	}
}

function make_item_box(item, source) {
	var image = $("<img>").attr("src", item.image ? API + "/" + item.image + ".png" : "icons/" + source.id + ".png")
		                  .attr("alt", item.image ? item.title : source.name)
		                  .attr("class", item.image ? "article-image" : "article-image-dummy")
		                  .on("load", set_image_class)
    var source_image = null
    if (item.image) {
    	var source_image = $("<img>").attr("src", "icons/" + source.id + ".png")
    	                             .attr("alt", source.name)
    	                             .attr("class", "source-image")
    }
	var imagebox = $("<div>").attr("class", "image")
	                         .append(image)
	if (source_image) {
		imagebox.append(source_image)
	}

	var headera = $("<a>").text(item.title)
	                      .attr("href", item.url)
	var header = $("<h2>").append(headera)
	var summary = $("<p>").text(item.summary)
	var content = $("<div>").attr("class", "content")
	                        .append(header)
	                        .append(summary)

	var article = $("<article>").attr("lang", source.lang)
	                            .attr("id", "article-" + item.id)
	                            .append(imagebox)
	                            .append(content)
	return article
}

function refresh_news_display() {
	var keys = []
	for (var key in GLOBAL.items) {
		keys.push(key)
	}
	keys.sort(function(a, b) {
		var akey = GLOBAL.sources[a].name.toLowerCase()
		var bkey = GLOBAL.sources[b].name.toLowerCase()
		return (akey < bkey ? -1 : (akey > bkey ? 1 : 0))
	})

	var i = 0
	var insert_anchor = $("#anchor")
	$("#items article").attr("data-delete", "data-delete")
	do {
		var has_hit = false
		var retry = 0
		for (var k = 0; k < keys.length; ++k) {
			var item = GLOBAL.items[keys[k]][i + retry]
			if (item) {
				has_hit = true

				var box = $("#article-" + item.id)
				if (box.length && !box.attr("data-delete")) {
					// article already in DOM because item is a duplicate
					// retry with next article
					--k
					++retry
					continue
				} else if (box.length) {
					// article already in DOM
					box.attr("data-delete", null)
				} else {
					// article not added yet
					box = make_item_box(item, GLOBAL.sources[keys[k]])
				}
				insert_anchor.before(box)
			}
			retry = 0
		}
		++i
	} while (has_hit)

	$("#items article[data-delete]").remove()

	$("#lastupdate").text(new Date().toLocaleString(
		window.navigator.userLanguage || window.navigator.language, {
		weekday: "long",
		year: "numeric",
		month: "long",
		day: "numeric",
		hour: "numeric",
		minute: "numeric",
	}))
}

{'fraction_non_alphanumeric': 0.1820593599734704, 'fraction_numerical': 0.002984579671696236, 'mean_word_length': 2.2447552447552446, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6292387', 'n_tokens_mistral': 2257, 'n_tokens_neox': 2039, 'n_words': 459}