<commit_before><!DOCTYPE html>
<html>
<head>
	<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
	<title>SEO Generator</title>
	<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.8.2/jquery.min.js"></script>
	<script type="text/javascript">
		/***************
		CONFIG
		***************/
		var domain = window.location.origin;
		var pages = [
			"/my/custom/page.html"
		];
		var prodBlacklist = [
			'WIKITEST'
			];
		var navcatBlacklist = [
			'.michael'
			];
		/***************
		SETUP
		***************/
		var apiDomain = domain + "/jsonapi/"
		var session = new Date().getTime();
		var requestNum = 0;
		//readable log, for informing the user without opening the console.  Used to list all the escaped fragment urls indexed.
		function rLog(msg){
			$('#output').append(msg+"<br/>");
		};
		function request(obj, callback){
			console.log(apiDomain);
			$.extend(obj, {
				"_clientid" : "zmvc",
				"_uuid" : requestNum,
				"_version" : "201352",
				"_session" : session
				});
			console.log(obj);
			var request = $.ajax({
				type: "POST",
				url: apiDomain,
				async: true,
				contentType : "application/json",
				dataType:"json",
				data : JSON.stringify(obj)
			})
			if(typeof callback == 'function'){
				request.success(callback);
			}
			requestNum++;
			
		};
		function fetchSEO(token){
			if(!window.lock){
				$('#seoTokenForm').hide();
				window.lock = true;
				window.token = token;
				
				//Temporarily bypassing this until we have a domain on 201401 to test
				var obj = {
					"_cmd" : "appSEOFetch",
					"token" : token
				}
				request(obj, function(response){
					var objects = response['@OBJECTS'];
					for(var i in objects){
						var o = objects[i];
						var p = buildPage(o);
						//Still use false? Blacklisted stuff comes out as false?
						if(p){
							pages.push(p);
						}
					}
					for(var i in pages){
						var paramBase = "?v="+new Date().getTime()+"&seoRequest=1&quiet=1";
					
						var p = pages[i];
						if(typeof p === "string"){
							pages[i] = {
								display : p,
								path : p,
								include : true
							}
							p = pages[i];
						}
						
						var paramStr = paramBase;
						if(p.uriParams){
							for(var i in p.uriParams){
								paramStr += "&"+i+"="+p.uriParams[i];
								}
							}
						p.path = domain+paramStr+(p.path ? "#!"+p.path : "");
						//console.log(p.path);
					}
					
					promptPages();
				});				
			}
		};
		//adds a page object to pages
		function buildPage(pageInfo){
			var page = false;
			switch(pageInfo.type){
				case "pid":
					page = {
						display : "Product["+pageInfo.id+"]",
						uriParams : {
							pageType : "product",
							pid : pageInfo.id
						},
						include : ($.inArray(pageInfo.id, prodBlacklist) < 0)
					}
					break;
				case "navcat" : 
					page = {
						display : "Category["+pageInfo.id+"]",
						uriParams : {
							pageType : "category",
							navcat : pageInfo.id
						},
						include : ($.inArray(pageInfo.id, navcatBlacklist) < 0)<commit_msg>app-seo - add check of pageInfo.noindex<commit_after><!DOCTYPE html>
<html>
<head>
	<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
	<title>SEO Generator</title>
	<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.8.2/jquery.min.js"></script>
	<script type="text/javascript">
		/***************
		CONFIG
		***************/
		var domain = window.location.origin;
		var pages = [
			"/my/custom/page.html"
		];
		var prodBlacklist = [
			'WIKITEST'
			];
		var navcatBlacklist = [
			'.michael'
			];
		/***************
		SETUP
		***************/
		var apiDomain = domain + "/jsonapi/"
		var session = new Date().getTime();
		var requestNum = 0;
		//readable log, for informing the user without opening the console.  Used to list all the escaped fragment urls indexed.
		function rLog(msg){
			$('#output').append(msg+"<br/>");
		};
		function request(obj, callback){
			console.log(apiDomain);
			$.extend(obj, {
				"_clientid" : "zmvc",
				"_uuid" : requestNum,
				"_version" : "201352",
				"_session" : session
				});
			console.log(obj);
			var request = $.ajax({
				type: "POST",
				url: apiDomain,
				async: true,
				contentType : "application/json",
				dataType:"json",
				data : JSON.stringify(obj)
			})
			if(typeof callback == 'function'){
				request.success(callback);
			}
			requestNum++;
			
		};
		function fetchSEO(token){
			if(!window.lock){
				$('#seoTokenForm').hide();
				window.lock = true;
				window.token = token;
				
				//Temporarily bypassing this until we have a domain on 201401 to test
				var obj = {
					"_cmd" : "appSEOFetch",
					"token" : token
				}
				request(obj, function(response){
					var objects = response['@OBJECTS'];
					for(var i in objects){
						var o = objects[i];
						var p = buildPage(o);
						//Still use false? Blacklisted stuff comes out as false?
						if(p){
							pages.push(p);
						}
					}
					for(var i in pages){
						var paramBase = "?v="+new Date().getTime()+"&seoRequest=1&quiet=1";
					
						var p = pages[i];
						if(typeof p === "string"){
							pages[i] = {
								display : p,
								path : p,
								include : true
							}
							p = pages[i];
						}
						
						var paramStr = paramBase;
						if(p.uriParams){
							for(var i in p.uriParams){
								paramStr += "&"+i+"="+p.uriParams[i];
								}
							}
						p.path = domain+paramStr+(p.path ? "#!"+p.path : "");
						//console.log(p.path);
					}
					
					promptPages();
				});				
			}
		};
		//adds a page object to pages
		function buildPage(pageInfo){
			var page = false;
			switch(pageInfo.type){
				case "pid":
					page = {
						display : "Product["+pageInfo.id+"]",
						uriParams : {
							pageType : "product",
							pid : pageInfo.id
						},
						include : ($.inArray(pageInfo.id, prodBlacklist) < 0) && !pageInfo.noindex
					}
					break;
				case "navcat" : 
					page = {
						display : "Category["+pageInfo.id+"]",
						uriParams : {
							pageType : "category",
							navcat : pageInfo.id
						},
						include : ($.inArray(pageInfo.id, navcatBlacklist) < 0) && !pageInfo.noindex
{'fraction_non_alphanumeric': 0.17858303542411438, 'fraction_numerical': 0.0068248293792655184, 'mean_word_length': 2.3324309691391445, 'pattern_counts': {'":': 2, '<': 27, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6362733', 'n_tokens_mistral': 2905, 'n_tokens_neox': 2061, 'n_words': 455}