<commit_before>//var wikipediaLink = "https://en.wikipedia.org/wiki/Beyonc%C3%A9";
var wikipediaLink = "https://en.wikipedia.org/wiki/Justin_Timberlake";
var contentArray = []
var myTableArray = [];




function makeArrayFromTable(){
	myTableArray = [];
	$("table tr").each(function() {
    var arrayOfThisRow = [];
    var tableData = $(this).find('td');
	var titel = $(this).find('th').text();
    if (tableData.length > 0) {
        tableData.each(function() { arrayOfThisRow.push($(this).text()); });
		var thisRow = [];
		thisRow[0] = titel;
		thisRow[1] = arrayOfThisRow;
		
        myTableArray.push(thisRow);
    }
});
console.log(myTableArray);
}

function removeBrackets(inputText){
	var startSplitter = "[";
	var endSplitter = "]";
	//console.log("removeBrackets")
	var firstRemoved = inputText.split(startSplitter);
	for(var i=0; i<firstRemoved.length; i++){
		var currentLineArray = firstRemoved[i].split(endSplitter);
		firstRemoved[i] = currentLineArray[currentLineArray.length-1]
	}
	var cleanText = firstRemoved.join();
	//console.log(cleanText);
	return cleanText;
}
function getTitleAndText(inputText){
	var headerArray = inputText.split('class="mw-headline"');
	//console.log(headerArray);
	var titelAndTextArray = [];
	for(var i =1; i<headerArray.length-3; i++){
		var titelAndText = headerArray[i].substr(headerArray[i].indexOf('>')+1);
		var titel = htmlToCleanText(titelAndText.substr(0,titelAndText.indexOf('</'))); 
		var text = htmlToCleanText(titelAndText.substr(titelAndText.indexOf('</')));
		var texts = text.split(".");
		titelAndTextArray.push([titel, texts]);
	}
	return titelAndTextArray;
}

function htmlToCleanText(htmlText){
	document.getElementById("tempDiv").innerHTML = htmlText;
	var cleanText = removeBrackets($("#tempDiv").text()).replace(/(\r\n|\n|\r)/gm,"");
	cleanText = cleanText.replace(/,/gi," COMMARHERE ");
	cleanText = cleanText.replace(/NSYNC/gi," N sync ");
	cleanText = cleanText.replace(/#/gi,"");
		cleanText = cleanText.replace(/"/gi,"");
			cleanText = cleanText.replace(/'/gi,"");
		cleanText = cleanText.replace(/:/gi,"");
	cleanText = cleanText.replace(/[()]/gi,"");
	//cleanText = cleanText.replace(")","");
	return cleanText;
}

function onload(){
		console.log("Laddar..");
		var theUrl = wikipediaLink;
		$.ajax({
			url: theUrl,
			type: 'GET',
			success: function(res) {
				var text = res.responseText;
				//console.log(text);
				document.getElementById("tempDiv").innerHTML = text;
				var content = document.getElementById("bodyContent").innerHTML;
				makeArrayFromTable();
				//console.log(content);
				//document.getElementById("container").innerHTML = content;
				contentArray = getTitleAndText(content);
				console.log(contentArray);
				document.getElementById("tempDiv").innerHTML = "";
				createTopic();
			}
		});
		

}
function creatQuestionText(){
	var dialogtxt = "";
	
	return dialogtxt;
}

function createTopic(){
	var dialogtxt = "";
	//dialogtxt = "topic: ~ronaldTalk () \n language: enu";
	//dialogtxt += "\nconcept:(music) [music artist]";
	dialogtxt += "\nu:(story) %storyMode Look at my stomach to see the headlines and tell me what you want to hear about?";
	for(var i = 0; i<contentArray.length; i++){
		var sentences = contentArray[i][1];
		var titel = contentArray[i][0];
		if(sentences.length>2){
			//rubriker här
			dialogtxt += "\n\tu1:("+titel+") Wikipedia says "+sentences[0].replace(/COMMARHERE/g," . ")+"";
			//tre första meningarna
			for(var j = 1; j<sentences.length && j<3; j++){
				var thisScentence = sentences[j];
				thisScentence = thisScentence.replace(/COMMARHERE/g," . ");
				dialogtxt += thisScentence+" . ";
			}
			
			dialogtxt +='  ^first[" $pauseOn==true $pause=1  A. A. A. "  " . "]  Do you want to hear more about this topic? ';
			dialogtxt += "\n\t\tu2:(~yes) ";
			for(var j = 3; j<sentences.length; j++){
				var thisScentence = sentences[j];
				thisScentence = thisScentence.replace(/COMMARHERE/g," . ");
				dialogtxt += thisScentence+" . ";
			}
			dialogtxt += "\n\t\tu2:(~no) ok  ^goto(storyMode) \n";
		}else{
			dialogtxt += "\n\tu1:("+titel+") Wikipedia says "+sentences[0].replace(/COMMARHERE/g," . ")+"";
			//tre första meningarna
			for(var j = 1; j<sentences.length && j<3; j++){<commit_msg>updated cleanText<commit_after>//var wikipediaLink = "https://en.wikipedia.org/wiki/Beyonc%C3%A9";
var wikipediaLink = "https://en.wikipedia.org/wiki/Justin_Timberlake";
var contentArray = []
var myTableArray = [];




function makeArrayFromTable(){
	myTableArray = [];
	$("table tr").each(function() {
    var arrayOfThisRow = [];
    var tableData = $(this).find('td');
	var titel = $(this).find('th').text();
    if (tableData.length > 0) {
        tableData.each(function() { arrayOfThisRow.push($(this).text()); });
		var thisRow = [];
		thisRow[0] = titel;
		thisRow[1] = arrayOfThisRow;
		
        myTableArray.push(thisRow);
    }
});
console.log(myTableArray);
}

function removeBrackets(inputText){
	var startSplitter = "[";
	var endSplitter = "]";
	//console.log("removeBrackets")
	var firstRemoved = inputText.split(startSplitter);
	for(var i=0; i<firstRemoved.length; i++){
		var currentLineArray = firstRemoved[i].split(endSplitter);
		firstRemoved[i] = currentLineArray[currentLineArray.length-1]
	}
	var cleanText = firstRemoved.join();
	//console.log(cleanText);
	return cleanText;
}
function getTitleAndText(inputText){
	var headerArray = inputText.split('class="mw-headline"');
	//console.log(headerArray);
	var titelAndTextArray = [];
	for(var i =1; i<headerArray.length-3; i++){
		var titelAndText = headerArray[i].substr(headerArray[i].indexOf('>')+1);
		var titel = htmlToCleanText(titelAndText.substr(0,titelAndText.indexOf('</'))); 
		var text = htmlToCleanText(titelAndText.substr(titelAndText.indexOf('</')));
		var texts = text.split(".");
		titelAndTextArray.push([titel, texts]);
	}
	return titelAndTextArray;
}

function htmlToCleanText(htmlText){
	document.getElementById("tempDiv").innerHTML = htmlText;
	var cleanText = removeBrackets($("#tempDiv").text()).replace(/(\r\n|\n|\r)/gm,"");
	cleanText = cleanText.replace(/,/gi," COMMARHERE ");
	cleanText = cleanText.replace(/NSYNC/gi," N sync ");
	cleanText = cleanText.replace(/#/gi,"");
	cleanText = cleanText.replace(/;/gi,"");
	cleanText = cleanText.replace(/$/gi,"");
	cleanText = cleanText.replace(/\//gi,"");
		cleanText = cleanText.replace(/"/gi,"");
			cleanText = cleanText.replace(/'/gi,"");
		cleanText = cleanText.replace(/:/gi,"");
	cleanText = cleanText.replace(/[()]/gi,"");
	//cleanText = cleanText.replace(")","");
	return cleanText;
}

function onload(){
		console.log("Laddar..");
		var theUrl = wikipediaLink;
		$.ajax({
			url: theUrl,
			type: 'GET',
			success: function(res) {
				var text = res.responseText;
				//console.log(text);
				document.getElementById("tempDiv").innerHTML = text;
				var content = document.getElementById("bodyContent").innerHTML;
				makeArrayFromTable();
				//console.log(content);
				//document.getElementById("container").innerHTML = content;
				contentArray = getTitleAndText(content);
				console.log(contentArray);
				document.getElementById("tempDiv").innerHTML = "";
				createTopic();
			}
		});
		

}
function creatQuestionText(){
	var dialogtxt = "";
	
	return dialogtxt;
}

function createTopic(){
	var dialogtxt = "";
	//dialogtxt = "topic: ~ronaldTalk () \n language: enu";
	//dialogtxt += "\nconcept:(music) [music artist]";
	dialogtxt += "\nu:(storyMode) %storyMode $openPage=story Look at my stomach to see the headlines and tell me what you want to hear about?";
	for(var i = 0; i<contentArray.length; i++){
		var sentences = contentArray[i][1];
		var titel = contentArray[i][0];
		if(sentences.length>2){
			//rubriker här
			dialogtxt += "\n\tu1:("+titel+") Wikipedia says "+sentences[0].replace(/COMMARHERE/g," . ")+"";
			//tre första meningarna
			for(var j = 1; j<sentences.length && j<3; j++){
				var thisScentence = sentences[j];
				thisScentence = thisScentence.replace(/COMMARHERE/g," . ");
				dialogtxt += thisScentence+" . ";
			}
			
			dialogtxt +='  ^first[" $pauseOn==true $pause=1  <NAME>. "  " . "]  Do you want to hear more about this topic? ';
			dialogtxt += "\n\t\tu2:(~yes) ";
			for(var j = 3; j<sentences.length; j++){
				var thisScentence = sentences[j];
				thisScentence = thisScentence.replace(/COMMARHERE/g," . ");
				dialogtxt += thisScentence+" . ";
			}
			dialogtxt += "\n\t\tu2:(~no) ok  ^goto(storyMode) \n";
		}else{
			dialogtxt += "\n\tu1:("+titel+") Wikipedia says "+sentences[0].replace(/COMMARHERE/g," . ")+"";
			//tre första meningarna
			for(var j = 1; j<sentences.length && j<3; j++){
{'fraction_non_alphanumeric': 0.18870108821486456, 'fraction_numerical': 0.0062514470942347765, 'mean_word_length': 4.3, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 10, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5084432', 'n_tokens_mistral': 3244, 'n_tokens_neox': 2955, 'n_words': 553}