<commit_before><html>
<body>
	iframe from cometd.
	<script>
		document.domain = "test.com";

		if (typeof XMLHttpRequest == "undefined") {
			XMLHttpRequest = function () {
				try { return new ActiveXObject("Msxml2.XMLHTTP.6.0"); }
				catch (e1) {}
				try { return new ActiveXObject("Msxml2.XMLHTTP.3.0"); }
				catch (e2) {}
				try { return new ActiveXObject("Msxml2.XMLHTTP"); }
				catch (e3) {}
				try { return new ActiveXObject("Microsoft.XMLHTTP"); }
				catch (e4) {}
				throw new Error("This browser does not support XMLHttpRequest.");
			};
		}

		function ajax(url) {
			var xhr = new XMLHttpRequest;
			xhr.open("get",url, true);
			xhr.send(null);
		}

		function CometClient(host){
			this.host = host;
			this.pos = 0;

			this.auth = function(uid, sid) {
				
				this.uid = uid;
				this.sid = sid;

				url = "http://"+this.host+"/meta/authenticate?uid="+this.uid+"&sid="+this.sid;
				ajax(url);
			}

			this.connect = function(channel, onMsg) {
				
				var xhr = new XMLHttpRequest;
				xhr.open("get", "http://"+this.host+"/meta/connect?name="+channel+"&uid="+this.uid+"&sid="+this.sid, true);
				var comet = this;
				xhr.onreadystatechange = function() {
					var data = xhr.responseText;
					if(xhr.readyState === 3 || xhr.readyState == 4) {

						// check if we've read anything before.
						var curLen = data.length;
						do {
							var msg = cutMessage(data.substr(comet.pos));
							// console.log("got: ", msg, "length=", msg.length);
							// console.log("following: ", data.substr(comet.pos + msg.length, 10));

							try {
								var obj = JSON.parse(msg);
								comet.pos += msg.length;
								switch(obj[0]) {
									case 'msg':
										if(obj[1].channel == channel) {
											onMsg(obj[1]);
										} else { // wrong channel, raise error? (TODO)

										}
										break;

									case 'meta':
										// TODO: handle service messages
										break;
								}
							} catch(e) {
								console.log(e);
								console.log(msg);
								comet.connect(channel, onMsg);
							}
						} while(comet.pos + msg.length != data.length);

					}
					if(xhr.readyState == 4) { // reconnect
						comet.connect(channel, onMsg);
					}
				};
				xhr.send(null);
			}
		};

		function cutMessage(msg) {
			if(msg[0] != '[') {
				return null;
			}
			var pos, depth = 0;
			var in_string = false;
			var len = msg.length;

			for(pos = 0; pos < len; pos++) {
				switch(msg[pos]) {
					case '"':
						in_string = !in_string;
						if(in_string) {
							continue;
						}
						break;
					case '[':
						depth++;
						break;
					case ']':
						depth--;
						break;
				}
				if(depth == 0) {
					return msg.substr(0, pos+1);
				}
			}

			return null;

		}
	</script>
	
	
</body>
</html>
<commit_msg>Cleanup, bugfix in iframe page.<commit_after><html>
<body>
	iframe from cometd.
	<script>
		document.domain = "test.com";

		if (typeof XMLHttpRequest == "undefined") {
			XMLHttpRequest = function () {
				try { return new ActiveXObject("Msxml2.XMLHTTP.6.0"); }
				catch (e1) {}
				try { return new ActiveXObject("Msxml2.XMLHTTP.3.0"); }
				catch (e2) {}
				try { return new ActiveXObject("Msxml2.XMLHTTP"); }
				catch (e3) {}
				try { return new ActiveXObject("Microsoft.XMLHTTP"); }
				catch (e4) {}
				throw new Error("This browser does not support XMLHttpRequest.");
			};
		}

		function ajax(url) {
			var xhr = new XMLHttpRequest;
			xhr.open("get",url, true);
			xhr.send(null);
		}

		function CometClient(host){
			this.host = host;
			this.pos = 0;

			this.auth = function(uid, sid) {
				
				this.uid = uid;
				this.sid = sid;

				url = "http://"+this.host+"/meta/authenticate?uid="+this.uid+"&sid="+this.sid;
				ajax(url);
			}

			this.connect = function(channel, onMsg, onMeta) {
				
				var xhr = new XMLHttpRequest;
				xhr.open("get", "http://"+this.host+"/meta/connect?name="+channel+"&uid="+this.uid+"&sid="+this.sid, true);
				var comet = this;
				xhr.onreadystatechange = function() {
					var data = xhr.responseText;
					if(xhr.readyState === 3 || xhr.readyState == 4) {

						// check if we've read anything before.
						var curLen = data.length;
						do {
							// this might only be the first part of our current packet.
							var msg = cutMessage(data.substr(comet.pos));

							try {
								var obj = JSON.parse(msg);
								comet.pos += msg.length; // add to what we've read so far.

								switch(obj[0]) {
									case 'msg': // regular message
										if(obj[1].channel == channel) {
											if(onMsg) onMsg(obj[1]);
										} // in the case of a wrong channel, raise error? (TODO)
										break;

									case 'meta': // TODO: handle service messages, meta-messages.
										if(onMeta) onMeta(obj[1]);
										break;
								}
							} catch(e) { // TODO: how do we handle errors?
								xhr = null; // close
								comet.connect(channel, onMsg);
							}
						// there might be more in this event: consume the whole string.
						} while(comet.pos + msg.length != data.length);

					}
					if(xhr.readyState == 4) { // reconnect
						comet.connect(channel, onMsg);
					}
				};
				xhr.send(null);
			}
		};

		/**
		 * Cuts the first part of a message and returns it. What's after is discarded.
		 * if msg is '[abc][def]', this function will return '[abc]' only.
		 * if msg is '["abc]ef"]gh', this function will return '["abc]ef"]': characters between quotes are preserved.
		 */
		function cutMessage(msg) {

			if(msg[0] != '[') {
				return "";
			}
			var pos, depth = 0;
			var in_string = false;
			var len = msg.length;

			for(pos = 0; pos < len; pos++) {
				if(msg[pos] == '"') {
					in_string = !in_string;
					continue;
				}
				if(in_string) { // do not consider characters that are part of a string.
					continue;
				}
				// opening and closing brackets, check for depth.
				if(msg[pos] == '[') {
					depth++;
				} else if(msg[pos] == ']') {
					depth--;
				}

				if(depth == 0) {
					return msg.substr(0, pos+1);
				}
			}

			return "";

		}
	</script>
	
	
</body>
</html>

{'fraction_non_alphanumeric': 0.16276385224274406, 'fraction_numerical': 0.008410290237467019, 'mean_word_length': 2.187073042564372, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '45135', 'n_tokens_mistral': 2889, 'n_tokens_neox': 2061, 'n_words': 488}