<commit_before>require("fs").readFile(__dirname + "/testpage.html", function(a,b){debug(b.toString("utf8"));});

var getReadableContent = require("../"),
	Parser = require("htmlparser2/lib/Parser.js"),
	Readability = require("../readabilitySAX.js");

function debug(data){
	var readable = new Readability(),
		parser = new Parser(readable);
	
	parser.parseComplete(data);
	
	var data = readable.getArticle();
	
	if(JSON.stringify(data) !== JSON.stringify(expected_data))
		throw Error("Didn't got expected output!");
	
	if(require("util").inspect(readable._docElements[0], false, 1/0).length !== 981869)
		throw Error("tree had false size!");
	
	if(Object.keys(readable._scannedLinks).length !== expected_links)
		throw Error("lost some links!");
	
	console.log("Passed!");
};

var expected_links = 5;

var expected_data = { title: 'Realtime Performance Visualizations using Node.js - How To Node',
  nextPage: 'page/2',
  textLength: 12071,
  score: 83,
  html: '<div class="clearfix" id="main"><a href="/243bfe84f43affd3244e1828d90a8dca7fcc34c4/heat-tracer" class="sha-version">Static Version</a><p>This article outlines how to create a realtime heatmap of your syscall latency using HTML5, some great node modules, and DTrace. It was inspired by talk that <NAME> and <NAME> gave on Joyent\'s cool cloud analytics tools. While specific, the code provided could easily be adapted to provide a heatmap of any type of aggregation Dtrace is capable of providing. </p>\n\n<h2>System Requirements</h2>\n\n<p>First thing\'s first, you\'re going to need a system with DTrace. This likely means Solaris (or one of its decedents), OS X, or a BSD variant.  There doesn\'t appear to be Dtrace available for Linux. </p>\n\n<h2>Security</h2>\n\n<p>Secondly, please be aware that at the time of writing the demo code contains a fairly substantial secruity vulnerabilty. Namely the d script is sent from the client with no authentication what so ever. If you bind to localhost this shouldn\'t be a big deal for a demo. Time permitting I intend to clean up the code.  </p>\n\n<h2>Dependencies</h2>\n\n<p>For this tutorial you\'ll also need:</p>\n\n<pre><code><span class="pln">node </span><span class="pun">-</span><span class="pln"> http</span><span class="pun">:</span><span class="com">//nodejs.org/#download (duh)</span><span class="pln"><br/>npm </span><span class="pun">-</span><span class="pln"> https</span><span class="pun">:</span><span class="com">//github.com/isaacs/npm (makes installing modules a breeze)</span><span class="pln"><br/>node</span><span class="pun">-</span><span class="pln">libdtrace </span><span class="pun">-</span><span class="pln"> https</span><span class="pun">:</span><span class="com">//github.com/bcantrill/node-libdtrace (provides dtrace functionality)</span><span class="pln"><br/></span><span class="typ">Socket</span><span class="pun">.</span><span class="pln">IO </span><span class="pun">-</span><span class="pln"> </span><span class="str">\'npm install socket.io\'</span><span class="pln"> </span><span class="pun">(</span><span class="pln">web sockets made easy</span><span class="pun">)</span><span class="pln"><br/></span></code></pre>\n\n<h2>Server</h2>\n\n<p>Now we\'re ready to start writing our web server: </p>\n\n<p><embed src="http://youtube.com/"></embed> This is just a test embed! </p>\n\n<div class="snippet"><a href="heat-tracer/heat_tracer.js" class="code-link">heat_tracer.js</a><pre><code><span class="kwd">var</span><span class="pln"> http </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">require</span><span class="pun">(</span><span class="str">\'http\'</span><span class="pun">);</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> libdtrace </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">require</span><span class="pun">(</span><span class="str">\'libdtrace\'</span><span class="pun">);</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> io </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">require</span><span class="pun">(</span><span class="str">\'socket.io\'</span><span class="pun">);</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> express </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">require</span><span class="pun">(</span><span class="str">\'express\'</span><span class="pun">);</span><span class="pln"><br/></span><span class="com">/* create our express server and prepare to serve javascript files in ./public <br/>*/</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> app </span><span class="pun">=</span><span class="pln"> express</span><span class="pun">.</span><span class="pln">createServer</span><span class="pun">();</span><span class="pln"><br/>app</span><span class="pun">.</span><span class="pln">configure</span><span class="pun">(</span><span class="kwd">function</span><span class="pun">(){</span><span class="pln"><br/>app</span><span class="pun">.</span><span class="kwd">use</span><span class="pun">(</span><span class="pln">express</span><span class="pun">.</span><span class="pln">staticProvider</span><span class="pun">(</span><span class="pln">__dirname </span><span class="pun">+</span><span class="pln"> </span><span class="str">\'/public\'</span><span class="pun">));</span><span class="pln"><br/></span><span class="pun">});</span><span class="pln"><br/></span><span class="com">/* Before we go any further we must realize that each time a user connects we\'re going to want to <br/>them send them dtrace aggregation every second. We can do so using \'setInterval\', but we must<br/>keep track of both the intervals we set and the dtrace consumers that are created as we\'ll need <br/>them later when the client disconnects. <br/>*/</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> interval_id_by_session_id </span><span class="pun">=</span><span class="pln"> </span><span class="pun">{};</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> dtp_by_session_id </span><span class="pun">=</span><span class="pln"> </span><span class="pun">{};</span><span class="pln"><br/></span><span class="com">/* In order to effecienctly send packets we\'re going to use the Socket.IO library which seemlessly <br/>integrates with express. &nbsp;<br/>*/</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> websocket_server </span><span class="pun">=</span><span class="pln"> io</span><span class="pun">.</span><span class="pln">listen</span><span class="pun">(</span><span class="pln">app</span><span class="pun">);</span><span class="pln"> <br/></span><span class="com">/* Now that we have a web socket server, we need to create a handler for connection events. These <br/>events represet a client connecting to our server */</span><span class="pln"><br/>websocket_server</span><span class="pun">.</span><span class="pln">on</span><span class="pun">(</span><span class="str">\'connection\'</span><span class="pun">,</span><span class="pln"> </span><span class="kwd">function</span><span class="pun">(</span><span class="pln">socket</span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"> <br/></span><span class="com">/* Like the web server object, we must also define handlers for various socket events that <br/>will happen during the lifetime of the connection. These will define how we interact with<br/>the client. The first is a message event which occurs when the client sends something to<br/>the server. */</span><span class="pln"><br/>socket</span><span class="pun">.</span><span class="pln">on</span><span class="pun">(</span><span class="pln"> </span><span class="str">\'message\'</span><span class="pun">,</span><span class="pln"> </span><span class="kwd">function</span><span class="pun">(</span><span class="pln">message</span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"> <br/></span><span class="com">/* The only message the client ever sends will be sent right after connecting. &nbsp;<br/>So it will happen only once during the lifetime of a socket. This message also <br/>contains a d script which defines an agregation to walk. <br/>*/</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> dtp </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">new</span><span class="pln"> libdtrace</span><span class="pun">.</span><span class="typ">Consumer</span><span class="pun">();</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> dscript </span><span class="pun">=</span><span class="pln"> message</span><span class="pun">[</span><span class="str">\'dscript\'</span><span class="pun">];</span><span class="pln"><br/>console</span><span class="pun">.</span><span class="pln">log</span><span class="pun">(</span><span class="pln"> dscript </span><span class="pun">);</span><span class="pln"><br/>dtp</span><span class="pun">.</span><span class="pln">strcompile</span><span class="pun">(</span><span class="pln">dscript</span><span class="pun">);</span><span class="pln"> &nbsp; &nbsp; &nbsp; &nbsp;<br/>dtp</span><span class="pun">.</span><span class="pln">go</span><span class="pun">();</span><span class="pln"><br/>dtp_by_session_id</span><span class="pun">[</span><span class="pln">socket</span><span class="pun">.</span><span class="pln">sessionId</span><span class="pun">]</span><span class="pln"> </span><span class="pun">=</span><span class="pln"> dtp</span><span class="pun">;</span><span class="pln"><br/></span><span class="com">/* All that\'s left to do is send the aggration data from the dscript. &nbsp;*/</span><span class="pln"><br/>interval_id_by_session_id</span><span class="pun">[</span><span class="pln">socket</span><span class="pun">.</span><span class="pln">sessionId</span><span class="pun">]</span><span class="pln"> </span><span class="pun">=</span><span class="pln"> setInterval</span><span class="pun">(</span><span class="kwd">function</span><span class="pln"> </span><span class="pun">()</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> aggdata </span><span class="pun">=</span><span class="pln"> </span><span class="pun">{};</span><span class="pln"><br/></span><span class="kwd">try</span><span class="pln"> </span><span class="pun">{</span><span class="pln"> <br/>dtp</span><span class="pun">.</span><span class="pln">aggwalk</span><span class="pun">(</span><span class="kwd">function</span><span class="pln"> </span><span class="pun">(</span><span class="pln">id</span><span class="pun">,</span><span class="pln"> key</span><span class="pun">,</span><span class="pln"> val</span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/></span><span class="kwd">for</span><span class="pun">(</span><span class="pln"> index </span><span class="kwd">in</span><span class="pln"> val </span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/></span><span class="com">/* console.log( \'key: \' + key + \', interval: \' + <br/>val[index][0][0] + \'-\' + val[index][0][1], \', count \' + val[index][1] ); */</span><span class="pln"><br/>aggdata</span><span class="pun">[</span><span class="pln">key</span><span class="pun">]</span><span class="pln"> </span><span class="pun">=</span><span class="pln"> val</span><span class="pun">;</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"> </span><span class="pun">);</span><span class="pln"><br/>socket</span><span class="pun">.</span><span class="pln">send</span><span class="pun">(</span><span class="pln"> aggdata </span><span class="pun">);</span><span class="pln"> <br/></span><span class="pun">}</span><span class="pln"> </span><span class="kwd">catch</span><span class="pun">(</span><span class="pln"> err </span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/>console</span><span class="pun">.</span><span class="pln">log</span><span class="pun">(</span><span class="pln">err</span><span class="pun">);</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="pun">},</span><span class="pln"> &nbsp;</span><span class="lit">1001</span><span class="pln"> </span><span class="pun">);</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"> </span><span class="pun">);</span><span class="pln"><br/></span><span class="com">/* Not so fast. If a client disconnects we don\'t want their respective dtrace consumer to <br/>keep collecting data any more. We also don\'t want to try to keep sending anything to them<br/>period. So clean up. */</span><span class="pln"><br/>socket</span><span class="pun">.</span><span class="pln">on</span><span class="pun">(</span><span class="str">\'disconnect\'</span><span class="pun">,</span><span class="pln"> </span><span class="kwd">function</span><span class="pun">(){</span><span class="pln"> <br/>clearInterval</span><span class="pun">(</span><span class="pln">clearInterval</span><span class="pun">(</span><span class="pln">interval_id_by_session_id</span><span class="pun">[</span><span class="pln">socket</span><span class="pun">.</span><span class="pln">sessionId</span><span class="pun">]));</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> dtp </span><span class="pun">=</span><span class="pln"> dtp_by_session_id</span><span class="pun">[</span><span class="pln">socket</span><span class="pun">.</span><span class="pln">sessionId</span><span class="pun">];</span><span class="pln"><br/></span><span class="kwd">delete</span><span class="pln"> dtp_by_session_id</span><span class="pun">[</span><span class="pln">socket</span><span class="pun">.</span><span class="pln">sessionId</span><span class="pun">];</span><span class="pln"> <br/>dtp</span><span class="pun">.</span><span class="pln">stop</span><span class="pun">();</span><span class="pln"> &nbsp; &nbsp; <br/>console</span><span class="pun">.</span><span class="pln">log</span><span class="pun">(</span><span class="str">\'disconnected\'</span><span class="pun">);</span><span class="pln"><br/></span><span class="pun">});</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"> </span><span class="pun">);</span><span class="pln"><br/>app</span><span class="pun">.</span><span class="pln">listen</span><span class="pun">(</span><span class="lit">80</span><span class="pun">);</span></code></pre></div>\n\n<h2>Client</h2>\n\n<p>In order to display our heatmap, we\'re going to need some basic HTML with a canvas element:</p>\n\n<div class="snippet"><a href="heat-tracer/public/heat_tracer.html" class="code-link">public/heat_tracer.html</a><pre><code><span class="tag">&lt;html&gt;</span><span class="pln"><br/></span><span class="tag">&lt;head&gt;</span><span class="pln"><br/></span><span class="tag">&lt;script</span><span class="pln"> </span><span class="atn">src</span><span class="pun">=</span><span class="atv">"http://localhost/socket.io/socket.io.js"</span><span class="tag">&gt;&lt;/script&gt;</span><span class="pln"> <br/></span><span class="tag">&lt;script</span><span class="pln"> </span><span class="atn">src</span><span class="pun">=</span><span class="atv">"http://localhost/heat_tracer_client.js"</span><span class="tag">&gt;&lt;/script&gt;</span><span class="pln"> <br/></span><span class="tag">&lt;/head&gt;</span><span class="pln"><br/></span><span class="tag">&lt;body</span><span class="pln"> </span><span class="atn">onLoad</span><span class="pun">=</span><span class="atv">\'</span><span class="pln">heat_tracer</span><span class="pun">()</span><span class="atv">\'</span><span class="tag">&gt;</span><span class="pln"><br/></span><span class="tag">&lt;canvas</span><span class="pln"> </span><span class="atn">id</span><span class="pun">=</span><span class="atv">\'canvas\'</span><span class="pln"> </span><span class="atn">width</span><span class="pun">=</span><span class="atv">\'1024\'</span><span class="pln"> </span><span class="atn">height</span><span class="pun">=</span><span class="atv">\'512\'</span><span class="tag">&gt;&lt;/canvas&gt;</span><span class="pln"><br/></span><span class="tag">&lt;/body&gt;</span><span class="pln"><br/></span><span class="tag">&lt;/html&gt;</span></code></pre></div>\n\n<p>Finally the JavaScript client which translates the raw  streaming data into pretty picture:</p>\n\n<div class="snippet"><a href="heat-tracer/public/heat_tracer_client.js" class="code-link">public/heat_tracer_client.js</a><pre><code><span class="com">/* On load we create our web socket (or flash socket if your browser doesn\'t support it ) and<br/>send the d script we wish to be tracing. This extremely powerful and *insecure*. */</span><span class="pln"><br/></span><span class="kwd">function</span><span class="pln"> heat_tracer</span><span class="pun">()</span><span class="pln"> </span><span class="pun">{</span><span class="pln"> <br/></span><span class="com">//Global vars</span><span class="pln"><br/>setup</span><span class="pun">();</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> socket </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">new</span><span class="pln"> io</span><span class="pun">.</span><span class="typ">Socket</span><span class="pun">(</span><span class="str">\'localhost\'</span><span class="pun">);</span><span class="pln"> </span><span class="com">//connect to localhost presently</span><span class="pln"><br/>socket</span><span class="pun">.</span><span class="pln">connect</span><span class="pun">();</span><span class="pln"><br/>socket</span><span class="pun">.</span><span class="pln">on</span><span class="pun">(</span><span class="str">\'connect\'</span><span class="pun">,</span><span class="pln"> </span><span class="kwd">function</span><span class="pun">(){</span><span class="pln"> <br/>console</span><span class="pun">.</span><span class="pln">log</span><span class="pun">(</span><span class="str">\'on connection\'</span><span class="pun">);</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> dscript </span><span class="pun">=</span><span class="pln"> </span><span class="str">"syscall:::entry\\n{\\nself-&gt;syscall_entry_ts[probefunc] = vtimestamp;\\n}\\nsyscall:::return\\n/self-&gt;syscall_entry_ts[probefunc]/\\n{\\n\\n@time[probefunc] = lquantize((vtimestamp - self-&gt;syscall_entry_ts[probefunc] ) / 1000, 0, 63, 2);\\nself-&gt;syscall_entry_ts[probefunc] = 0;\\n}"</span><span class="pun">;</span><span class="pln"><br/>socket</span><span class="pun">.</span><span class="pln">send</span><span class="pun">(</span><span class="pln"> </span><span class="pun">{</span><span class="pln"> </span><span class="str">\'dscript\'</span><span class="pln"> </span><span class="pun">:</span><span class="pln"> dscript </span><span class="pun">}</span><span class="pln"> </span><span class="pun">);</span><span class="pln"><br/></span><span class="pun">});</span><span class="pln"><br/></span><span class="com">/* The only messages we recieve should contain contain the dtrace aggregation data we requested<br/>on connection. */</span><span class="pln"><br/>socket</span><span class="pun">.</span><span class="pln">on</span><span class="pun">(</span><span class="str">\'message\'</span><span class="pun">,</span><span class="pln"> </span><span class="kwd">function</span><span class="pun">(</span><span class="pln">message</span><span class="pun">){</span><span class="pln"> <br/></span><span class="com">//console.log( message );</span><span class="pln"><br/>draw</span><span class="pun">(</span><span class="pln">message</span><span class="pun">);</span><span class="pln"><br/></span><span class="com">/* for ( key in message ) {<br/>val = message[key];<br/>console.log( \'key: \' + key + \', interval: \' + val[0][0] + \'-\' + val[0][1], \', count \' + val[1] );<br/>} &nbsp;<br/>*/</span><span class="pln"><br/></span><span class="pun">});</span><span class="pln"><br/>socket</span><span class="pun">.</span><span class="pln">on</span><span class="pun">(</span><span class="str">\'disconnect\'</span><span class="pun">,</span><span class="pln"> </span><span class="kwd">function</span><span class="pun">(){</span><span class="pln"> <br/></span><span class="pun">});</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="com">/* Take the aggregation data and update the heatmap */</span><span class="pln"><br/></span><span class="kwd">function</span><span class="pln"> draw</span><span class="pun">(</span><span class="pln">message</span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"> &nbsp;<br/></span><span class="com">/* Latest data goes in the right most column, initialize it */</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> syscalls_by_latency </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[];</span><span class="pln"><br/></span><span class="kwd">for</span><span class="pln"> </span><span class="pun">(</span><span class="pln"> </span><span class="kwd">var</span><span class="pln"> index </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span><span class="pun">;</span><span class="pln"> index </span><span class="pun">&lt;</span><span class="pln"> </span><span class="lit">32</span><span class="pun">;</span><span class="pln"> index</span><span class="pun">++</span><span class="pln"> </span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/>syscalls_by_latency</span><span class="pun">[</span><span class="pln">index</span><span class="pun">]</span><span class="pln"> </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span><span class="pun">;</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="com">/* Presently we have the latency for each system call quantized in our message. Merge the data<br/>such that we have all the system call latency quantized together. This gives us the number<br/>of syscalls made with latencies in each particular band. */</span><span class="pln"><br/></span><span class="kwd">for</span><span class="pln"> </span><span class="pun">(</span><span class="pln"> </span><span class="kwd">var</span><span class="pln"> syscall </span><span class="kwd">in</span><span class="pln"> message </span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> val </span><span class="pun">=</span><span class="pln"> message</span><span class="pun">[</span><span class="pln">syscall</span><span class="pun">];</span><span class="pln"><br/></span><span class="kwd">for</span><span class="pln"> </span><span class="pun">(</span><span class="pln"> result_index </span><span class="kwd">in</span><span class="pln"> val </span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> latency_start </span><span class="pun">=</span><span class="pln"> val</span><span class="pun">[</span><span class="pln">result_index</span><span class="pun">][</span><span class="lit">0</span><span class="pun">][</span><span class="lit">0</span><span class="pun">];</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> count </span><span class="pun">=</span><span class="pln"> &nbsp;val</span><span class="pun">[</span><span class="pln">result_index</span><span class="pun">][</span><span class="lit">1</span><span class="pun">];</span><span class="pln"><br/></span><span class="com">/* The d script we\'re using lquantizes from 0 to 63 in steps of two. So dividing by 2 <br/>tells us which row this result belongs in */</span><span class="pln"><br/>syscalls_by_latency</span><span class="pun">[</span><span class="typ">Math</span><span class="pun">.</span><span class="pln">floor</span><span class="pun">(</span><span class="pln">latency_start</span><span class="pun">/</span><span class="lit">2</span><span class="pun">)]</span><span class="pln"> </span><span class="pun">+=</span><span class="pln"> count</span><span class="pun">;</span><span class="pln"> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="com">/* We just created a new column, shift the console to the left and add it. */</span><span class="pln"><br/>console_columns</span><span class="pun">.</span><span class="pln">shift</span><span class="pun">();</span><span class="pln"><br/>console_columns</span><span class="pun">.</span><span class="pln">push</span><span class="pun">(</span><span class="pln">syscalls_by_latency</span><span class="pun">);</span><span class="pln"><br/>drawArray</span><span class="pun">(</span><span class="pln">console_columns</span><span class="pun">);</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="com">/* Draw the columns and rows that map up the heatmap on to the canvas element */</span><span class="pln"><br/></span><span class="kwd">function</span><span class="pln"> drawArray</span><span class="pun">(</span><span class="pln">console_columns</span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> canvas </span><span class="pun">=</span><span class="pln"> document</span><span class="pun">.</span><span class="pln">getElementById</span><span class="pun">(</span><span class="str">\'canvas\'</span><span class="pun">);</span><span class="pln"><br/></span><span class="kwd">if</span><span class="pln"> </span><span class="pun">(</span><span class="pln">canvas</span><span class="pun">.</span><span class="pln">getContext</span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> ctx </span><span class="pun">=</span><span class="pln"> canvas</span><span class="pun">.</span><span class="pln">getContext</span><span class="pun">(</span><span class="str">\'2d\'</span><span class="pun">);</span><span class="pln"> &nbsp;<br/></span><span class="kwd">for</span><span class="pln"> </span><span class="pun">(</span><span class="pln"> </span><span class="kwd">var</span><span class="pln"> column_index </span><span class="kwd">in</span><span class="pln"> console_columns </span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> column </span><span class="pun">=</span><span class="pln"> console_columns</span><span class="pun">[</span><span class="pln">column_index</span><span class="pun">];</span><span class="pln"> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <br/></span><span class="kwd">for</span><span class="pln"> </span><span class="pun">(</span><span class="pln"> </span><span class="kwd">var</span><span class="pln"> entry_index </span><span class="kwd">in</span><span class="pln"> column </span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/>entry </span><span class="pun">=</span><span class="pln"> column</span><span class="pun">[</span><span class="pln">entry_index</span><span class="pun">];</span><span class="pln"><br/></span><span class="com">/* We\'re using a logarithmic scale for the brightness. This was all arrived at by<br/>trial and error and found to work well on my Mac. &nbsp;In the future this <br/>could all be adjustable with controls */</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> red_value </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span><span class="pun">;</span><span class="pln"> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<br/></span><span class="kwd">if</span><span class="pln"> </span><span class="pun">(</span><span class="pln"> entry </span><span class="pun">!=</span><span class="pln"> </span><span class="lit">0</span><span class="pln"> </span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/>red_value </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Math</span><span class="pun">.</span><span class="pln">floor</span><span class="pun">(</span><span class="typ">Math</span><span class="pun">.</span><span class="pln">log</span><span class="pun">(</span><span class="pln">entry</span><span class="pun">)/</span><span class="typ">Math</span><span class="pun">.</span><span class="pln">log</span><span class="pun">(</span><span class="lit">2</span><span class="pun">));</span><span class="pln"> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="com">//console.log(red_value); &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </span><span class="pln"><br/>ctx</span><span class="pun">.</span><span class="pln">fillStyle </span><span class="pun">=</span><span class="pln"> </span><span class="str">\'rgb(\'</span><span class="pln"> </span><span class="pun">+</span><span class="pln"> </span><span class="pun">(</span><span class="pln">red_value </span><span class="pun">*</span><span class="pln"> </span><span class="lit">25</span><span class="pun">)</span><span class="pln"> </span><span class="pun">+</span><span class="pln"> </span><span class="str">\',0,0)\'</span><span class="pun">;</span><span class="pln"><br/>ctx</span><span class="pun">.</span><span class="pln">fillRect</span><span class="pun">(</span><span class="pln">column_index</span><span class="pun">*</span><span class="lit">16</span><span class="pun">,</span><span class="pln"> </span><span class="lit">496</span><span class="pun">-(</span><span class="pln">entry_index</span><span class="pun">*</span><span class="lit">16</span><span class="pun">),</span><span class="pln"> </span><span class="lit">16</span><span class="pun">,</span><span class="pln"> </span><span class="lit">16</span><span class="pun">);</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"> <br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="com">/* The heatmap is is really a 64x32 grid. Initialize the array which contains the grid data. */</span><span class="pln"><br/></span><span class="kwd">function</span><span class="pln"> setup</span><span class="pun">()</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/>console_columns </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[];</span><span class="pln"><br/></span><span class="kwd">for</span><span class="pln"> </span><span class="pun">(</span><span class="pln"> </span><span class="kwd">var</span><span class="pln"> column_index </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span><span class="pun">;</span><span class="pln"> column_index </span><span class="pun">&lt;</span><span class="pln"> </span><span class="lit">64</span><span class="pun">;</span><span class="pln"> column_index</span><span class="pun">++</span><span class="pln"> </span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> column </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[];</span><span class="pln"><br/></span><span class="kwd">for</span><span class="pln"> </span><span class="pun">(</span><span class="pln"> </span><span class="kwd">var</span><span class="pln"> entry_index </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span><span class="pun">;</span><span class="pln"> entry_index </span><span class="pun">&lt;</span><span class="pln"> </span><span class="lit">32</span><span class="pun">;</span><span class="pln"> entry_index</span><span class="pun">++</span><span class="pln"> </span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"> <br/>column</span><span class="pun">[</span><span class="pln">entry_index</span><span class="pun">]</span><span class="pln"> </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span><span class="pun">;</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"><br/>console_columns</span><span class="pun">.</span><span class="pln">push</span><span class="pun">(</span><span class="pln">column</span><span class="pun">);</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="pun">}</span></code></pre></div>\n\n<h2>Run It!</h2>\n\n<p>Run Heat Tacer with the following. Note, sudo is required by dtrace as it does kernal magic.</p>\n\n<pre><code><span class="pln">sudo node heat_tracer</span><span class="pun">.</span><span class="pln">js<br/></span></code></pre>\n\n<p>If all goes well you should see something a moving version of something like the image below.</p>\n\n<blockquote>\n  <p><img src="http://howtonode.org/heat-tracer/heat_tracer.png" alt="Alt value of image" title=""></img> </p>\n</blockquote>\n\n<h2>Contribute</h2>\n\n<p>You can find the latest version of Heat Tracer <a href="https://github.com/gflarity/Heat-Tracer">here</a>. It is my hope that this article will provide the ground work for a much more abitious performance analytics project. If you\'re interested in contributing please let me know.</p>\n\n<h2>Further Research</h2>\n\n<p>More information about Bryan and Brendan\'s demo can be found <a href="http://dtrace.org/blogs/brendan/2011/01/24/cloud-analytics-first-video/">here</a>.</p>\n\n<p>Socket.IO can be found <a href="http://socket.io/">here</a>.</p><hr></hr>\n\n<a href="http://disqus.com/forums/howtonodeorg/?url=ref">View the discussion thread.</a><a href="http://disqus.com" class="dsq-brlink">blog comments powered by</a>\n\n</div>' };<commit_msg>Added settings to resolve links<commit_after>require("fs").readFile(__dirname + "/testpage.html", function(a,b){debug(b.toString("utf8"));});

var getReadableContent = require("../"),
	Parser = require("htmlparser2/lib/Parser.js"),
	Readability = require("../readabilitySAX.js");

function debug(data){
	var readable = new Readability({
		pageURL: "http://howtonode.org/heat-tracer",
		resolvePaths: true
	}),
		parser = new Parser(readable);
	
	parser.parseComplete(data);
	
	var data = readable.getArticle();
	
	if(JSON.stringify(data) !== JSON.stringify(expected_data))
		throw Error("Didn't got expected output!");
	
	if(require("util").inspect(readable._docElements[0], false, 1/0).length !== 981869)
		throw Error("tree had false size!");
	
	if(Object.keys(readable._scannedLinks).length !== expected_links)
		throw Error("lost some links!");
	
	console.log("Passed!");
};

var expected_links = 5;

var expected_data = { title: 'Realtime Performance Visualizations using Node.js - How To Node',
  nextPage: 'page/2',
  textLength: 12071,
  score: 83,
  html: '<div class="clearfix" id="main"><a href="/243bfe84f43affd3244e1828d90a8dca7fcc34c4/heat-tracer" class="sha-version">Static Version</a><p>This article outlines how to create a realtime heatmap of your syscall latency using HTML5, some great node modules, and DTrace. It was inspired by talk that <NAME> and <NAME> gave on Joyent\'s cool cloud analytics tools. While specific, the code provided could easily be adapted to provide a heatmap of any type of aggregation Dtrace is capable of providing. </p>\n\n<h2>System Requirements</h2>\n\n<p>First thing\'s first, you\'re going to need a system with DTrace. This likely means Solaris (or one of its decedents), OS X, or a BSD variant.  There doesn\'t appear to be Dtrace available for Linux. </p>\n\n<h2>Security</h2>\n\n<p>Secondly, please be aware that at the time of writing the demo code contains a fairly substantial secruity vulnerabilty. Namely the d script is sent from the client with no authentication what so ever. If you bind to localhost this shouldn\'t be a big deal for a demo. Time permitting I intend to clean up the code.  </p>\n\n<h2>Dependencies</h2>\n\n<p>For this tutorial you\'ll also need:</p>\n\n<pre><code><span class="pln">node </span><span class="pun">-</span><span class="pln"> http</span><span class="pun">:</span><span class="com">//nodejs.org/#download (duh)</span><span class="pln"><br/>npm </span><span class="pun">-</span><span class="pln"> https</span><span class="pun">:</span><span class="com">//github.com/isaacs/npm (makes installing modules a breeze)</span><span class="pln"><br/>node</span><span class="pun">-</span><span class="pln">libdtrace </span><span class="pun">-</span><span class="pln"> https</span><span class="pun">:</span><span class="com">//github.com/bcantrill/node-libdtrace (provides dtrace functionality)</span><span class="pln"><br/></span><span class="typ">Socket</span><span class="pun">.</span><span class="pln">IO </span><span class="pun">-</span><span class="pln"> </span><span class="str">\'npm install socket.io\'</span><span class="pln"> </span><span class="pun">(</span><span class="pln">web sockets made easy</span><span class="pun">)</span><span class="pln"><br/></span></code></pre>\n\n<h2>Server</h2>\n\n<p>Now we\'re ready to start writing our web server: </p>\n\n<p><embed src="http://youtube.com/"></embed> This is just a test embed! </p>\n\n<div class="snippet"><a href="heat-tracer/heat_tracer.js" class="code-link">heat_tracer.js</a><pre><code><span class="kwd">var</span><span class="pln"> http </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">require</span><span class="pun">(</span><span class="str">\'http\'</span><span class="pun">);</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> libdtrace </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">require</span><span class="pun">(</span><span class="str">\'libdtrace\'</span><span class="pun">);</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> io </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">require</span><span class="pun">(</span><span class="str">\'socket.io\'</span><span class="pun">);</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> express </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">require</span><span class="pun">(</span><span class="str">\'express\'</span><span class="pun">);</span><span class="pln"><br/></span><span class="com">/* create our express server and prepare to serve javascript files in ./public <br/>*/</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> app </span><span class="pun">=</span><span class="pln"> express</span><span class="pun">.</span><span class="pln">createServer</span><span class="pun">();</span><span class="pln"><br/>app</span><span class="pun">.</span><span class="pln">configure</span><span class="pun">(</span><span class="kwd">function</span><span class="pun">(){</span><span class="pln"><br/>app</span><span class="pun">.</span><span class="kwd">use</span><span class="pun">(</span><span class="pln">express</span><span class="pun">.</span><span class="pln">staticProvider</span><span class="pun">(</span><span class="pln">__dirname </span><span class="pun">+</span><span class="pln"> </span><span class="str">\'/public\'</span><span class="pun">));</span><span class="pln"><br/></span><span class="pun">});</span><span class="pln"><br/></span><span class="com">/* Before we go any further we must realize that each time a user connects we\'re going to want to <br/>them send them dtrace aggregation every second. We can do so using \'setInterval\', but we must<br/>keep track of both the intervals we set and the dtrace consumers that are created as we\'ll need <br/>them later when the client disconnects. <br/>*/</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> interval_id_by_session_id </span><span class="pun">=</span><span class="pln"> </span><span class="pun">{};</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> dtp_by_session_id </span><span class="pun">=</span><span class="pln"> </span><span class="pun">{};</span><span class="pln"><br/></span><span class="com">/* In order to effecienctly send packets we\'re going to use the Socket.IO library which seemlessly <br/>integrates with express. &nbsp;<br/>*/</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> websocket_server </span><span class="pun">=</span><span class="pln"> io</span><span class="pun">.</span><span class="pln">listen</span><span class="pun">(</span><span class="pln">app</span><span class="pun">);</span><span class="pln"> <br/></span><span class="com">/* Now that we have a web socket server, we need to create a handler for connection events. These <br/>events represet a client connecting to our server */</span><span class="pln"><br/>websocket_server</span><span class="pun">.</span><span class="pln">on</span><span class="pun">(</span><span class="str">\'connection\'</span><span class="pun">,</span><span class="pln"> </span><span class="kwd">function</span><span class="pun">(</span><span class="pln">socket</span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"> <br/></span><span class="com">/* Like the web server object, we must also define handlers for various socket events that <br/>will happen during the lifetime of the connection. These will define how we interact with<br/>the client. The first is a message event which occurs when the client sends something to<br/>the server. */</span><span class="pln"><br/>socket</span><span class="pun">.</span><span class="pln">on</span><span class="pun">(</span><span class="pln"> </span><span class="str">\'message\'</span><span class="pun">,</span><span class="pln"> </span><span class="kwd">function</span><span class="pun">(</span><span class="pln">message</span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"> <br/></span><span class="com">/* The only message the client ever sends will be sent right after connecting. &nbsp;<br/>So it will happen only once during the lifetime of a socket. This message also <br/>contains a d script which defines an agregation to walk. <br/>*/</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> dtp </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">new</span><span class="pln"> libdtrace</span><span class="pun">.</span><span class="typ">Consumer</span><span class="pun">();</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> dscript </span><span class="pun">=</span><span class="pln"> message</span><span class="pun">[</span><span class="str">\'dscript\'</span><span class="pun">];</span><span class="pln"><br/>console</span><span class="pun">.</span><span class="pln">log</span><span class="pun">(</span><span class="pln"> dscript </span><span class="pun">);</span><span class="pln"><br/>dtp</span><span class="pun">.</span><span class="pln">strcompile</span><span class="pun">(</span><span class="pln">dscript</span><span class="pun">);</span><span class="pln"> &nbsp; &nbsp; &nbsp; &nbsp;<br/>dtp</span><span class="pun">.</span><span class="pln">go</span><span class="pun">();</span><span class="pln"><br/>dtp_by_session_id</span><span class="pun">[</span><span class="pln">socket</span><span class="pun">.</span><span class="pln">sessionId</span><span class="pun">]</span><span class="pln"> </span><span class="pun">=</span><span class="pln"> dtp</span><span class="pun">;</span><span class="pln"><br/></span><span class="com">/* All that\'s left to do is send the aggration data from the dscript. &nbsp;*/</span><span class="pln"><br/>interval_id_by_session_id</span><span class="pun">[</span><span class="pln">socket</span><span class="pun">.</span><span class="pln">sessionId</span><span class="pun">]</span><span class="pln"> </span><span class="pun">=</span><span class="pln"> setInterval</span><span class="pun">(</span><span class="kwd">function</span><span class="pln"> </span><span class="pun">()</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> aggdata </span><span class="pun">=</span><span class="pln"> </span><span class="pun">{};</span><span class="pln"><br/></span><span class="kwd">try</span><span class="pln"> </span><span class="pun">{</span><span class="pln"> <br/>dtp</span><span class="pun">.</span><span class="pln">aggwalk</span><span class="pun">(</span><span class="kwd">function</span><span class="pln"> </span><span class="pun">(</span><span class="pln">id</span><span class="pun">,</span><span class="pln"> key</span><span class="pun">,</span><span class="pln"> val</span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/></span><span class="kwd">for</span><span class="pun">(</span><span class="pln"> index </span><span class="kwd">in</span><span class="pln"> val </span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/></span><span class="com">/* console.log( \'key: \' + key + \', interval: \' + <br/>val[index][0][0] + \'-\' + val[index][0][1], \', count \' + val[index][1] ); */</span><span class="pln"><br/>aggdata</span><span class="pun">[</span><span class="pln">key</span><span class="pun">]</span><span class="pln"> </span><span class="pun">=</span><span class="pln"> val</span><span class="pun">;</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"> </span><span class="pun">);</span><span class="pln"><br/>socket</span><span class="pun">.</span><span class="pln">send</span><span class="pun">(</span><span class="pln"> aggdata </span><span class="pun">);</span><span class="pln"> <br/></span><span class="pun">}</span><span class="pln"> </span><span class="kwd">catch</span><span class="pun">(</span><span class="pln"> err </span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/>console</span><span class="pun">.</span><span class="pln">log</span><span class="pun">(</span><span class="pln">err</span><span class="pun">);</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="pun">},</span><span class="pln"> &nbsp;</span><span class="lit">1001</span><span class="pln"> </span><span class="pun">);</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"> </span><span class="pun">);</span><span class="pln"><br/></span><span class="com">/* Not so fast. If a client disconnects we don\'t want their respective dtrace consumer to <br/>keep collecting data any more. We also don\'t want to try to keep sending anything to them<br/>period. So clean up. */</span><span class="pln"><br/>socket</span><span class="pun">.</span><span class="pln">on</span><span class="pun">(</span><span class="str">\'disconnect\'</span><span class="pun">,</span><span class="pln"> </span><span class="kwd">function</span><span class="pun">(){</span><span class="pln"> <br/>clearInterval</span><span class="pun">(</span><span class="pln">clearInterval</span><span class="pun">(</span><span class="pln">interval_id_by_session_id</span><span class="pun">[</span><span class="pln">socket</span><span class="pun">.</span><span class="pln">sessionId</span><span class="pun">]));</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> dtp </span><span class="pun">=</span><span class="pln"> dtp_by_session_id</span><span class="pun">[</span><span class="pln">socket</span><span class="pun">.</span><span class="pln">sessionId</span><span class="pun">];</span><span class="pln"><br/></span><span class="kwd">delete</span><span class="pln"> dtp_by_session_id</span><span class="pun">[</span><span class="pln">socket</span><span class="pun">.</span><span class="pln">sessionId</span><span class="pun">];</span><span class="pln"> <br/>dtp</span><span class="pun">.</span><span class="pln">stop</span><span class="pun">();</span><span class="pln"> &nbsp; &nbsp; <br/>console</span><span class="pun">.</span><span class="pln">log</span><span class="pun">(</span><span class="str">\'disconnected\'</span><span class="pun">);</span><span class="pln"><br/></span><span class="pun">});</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"> </span><span class="pun">);</span><span class="pln"><br/>app</span><span class="pun">.</span><span class="pln">listen</span><span class="pun">(</span><span class="lit">80</span><span class="pun">);</span></code></pre></div>\n\n<h2>Client</h2>\n\n<p>In order to display our heatmap, we\'re going to need some basic HTML with a canvas element:</p>\n\n<div class="snippet"><a href="heat-tracer/public/heat_tracer.html" class="code-link">public/heat_tracer.html</a><pre><code><span class="tag">&lt;html&gt;</span><span class="pln"><br/></span><span class="tag">&lt;head&gt;</span><span class="pln"><br/></span><span class="tag">&lt;script</span><span class="pln"> </span><span class="atn">src</span><span class="pun">=</span><span class="atv">"http://localhost/socket.io/socket.io.js"</span><span class="tag">&gt;&lt;/script&gt;</span><span class="pln"> <br/></span><span class="tag">&lt;script</span><span class="pln"> </span><span class="atn">src</span><span class="pun">=</span><span class="atv">"http://localhost/heat_tracer_client.js"</span><span class="tag">&gt;&lt;/script&gt;</span><span class="pln"> <br/></span><span class="tag">&lt;/head&gt;</span><span class="pln"><br/></span><span class="tag">&lt;body</span><span class="pln"> </span><span class="atn">onLoad</span><span class="pun">=</span><span class="atv">\'</span><span class="pln">heat_tracer</span><span class="pun">()</span><span class="atv">\'</span><span class="tag">&gt;</span><span class="pln"><br/></span><span class="tag">&lt;canvas</span><span class="pln"> </span><span class="atn">id</span><span class="pun">=</span><span class="atv">\'canvas\'</span><span class="pln"> </span><span class="atn">width</span><span class="pun">=</span><span class="atv">\'1024\'</span><span class="pln"> </span><span class="atn">height</span><span class="pun">=</span><span class="atv">\'512\'</span><span class="tag">&gt;&lt;/canvas&gt;</span><span class="pln"><br/></span><span class="tag">&lt;/body&gt;</span><span class="pln"><br/></span><span class="tag">&lt;/html&gt;</span></code></pre></div>\n\n<p>Finally the JavaScript client which translates the raw  streaming data into pretty picture:</p>\n\n<div class="snippet"><a href="heat-tracer/public/heat_tracer_client.js" class="code-link">public/heat_tracer_client.js</a><pre><code><span class="com">/* On load we create our web socket (or flash socket if your browser doesn\'t support it ) and<br/>send the d script we wish to be tracing. This extremely powerful and *insecure*. */</span><span class="pln"><br/></span><span class="kwd">function</span><span class="pln"> heat_tracer</span><span class="pun">()</span><span class="pln"> </span><span class="pun">{</span><span class="pln"> <br/></span><span class="com">//Global vars</span><span class="pln"><br/>setup</span><span class="pun">();</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> socket </span><span class="pun">=</span><span class="pln"> </span><span class="kwd">new</span><span class="pln"> io</span><span class="pun">.</span><span class="typ">Socket</span><span class="pun">(</span><span class="str">\'localhost\'</span><span class="pun">);</span><span class="pln"> </span><span class="com">//connect to localhost presently</span><span class="pln"><br/>socket</span><span class="pun">.</span><span class="pln">connect</span><span class="pun">();</span><span class="pln"><br/>socket</span><span class="pun">.</span><span class="pln">on</span><span class="pun">(</span><span class="str">\'connect\'</span><span class="pun">,</span><span class="pln"> </span><span class="kwd">function</span><span class="pun">(){</span><span class="pln"> <br/>console</span><span class="pun">.</span><span class="pln">log</span><span class="pun">(</span><span class="str">\'on connection\'</span><span class="pun">);</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> dscript </span><span class="pun">=</span><span class="pln"> </span><span class="str">"syscall:::entry\\n{\\nself-&gt;syscall_entry_ts[probefunc] = vtimestamp;\\n}\\nsyscall:::return\\n/self-&gt;syscall_entry_ts[probefunc]/\\n{\\n\\n@time[probefunc] = lquantize((vtimestamp - self-&gt;syscall_entry_ts[probefunc] ) / 1000, 0, 63, 2);\\nself-&gt;syscall_entry_ts[probefunc] = 0;\\n}"</span><span class="pun">;</span><span class="pln"><br/>socket</span><span class="pun">.</span><span class="pln">send</span><span class="pun">(</span><span class="pln"> </span><span class="pun">{</span><span class="pln"> </span><span class="str">\'dscript\'</span><span class="pln"> </span><span class="pun">:</span><span class="pln"> dscript </span><span class="pun">}</span><span class="pln"> </span><span class="pun">);</span><span class="pln"><br/></span><span class="pun">});</span><span class="pln"><br/></span><span class="com">/* The only messages we recieve should contain contain the dtrace aggregation data we requested<br/>on connection. */</span><span class="pln"><br/>socket</span><span class="pun">.</span><span class="pln">on</span><span class="pun">(</span><span class="str">\'message\'</span><span class="pun">,</span><span class="pln"> </span><span class="kwd">function</span><span class="pun">(</span><span class="pln">message</span><span class="pun">){</span><span class="pln"> <br/></span><span class="com">//console.log( message );</span><span class="pln"><br/>draw</span><span class="pun">(</span><span class="pln">message</span><span class="pun">);</span><span class="pln"><br/></span><span class="com">/* for ( key in message ) {<br/>val = message[key];<br/>console.log( \'key: \' + key + \', interval: \' + val[0][0] + \'-\' + val[0][1], \', count \' + val[1] );<br/>} &nbsp;<br/>*/</span><span class="pln"><br/></span><span class="pun">});</span><span class="pln"><br/>socket</span><span class="pun">.</span><span class="pln">on</span><span class="pun">(</span><span class="str">\'disconnect\'</span><span class="pun">,</span><span class="pln"> </span><span class="kwd">function</span><span class="pun">(){</span><span class="pln"> <br/></span><span class="pun">});</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="com">/* Take the aggregation data and update the heatmap */</span><span class="pln"><br/></span><span class="kwd">function</span><span class="pln"> draw</span><span class="pun">(</span><span class="pln">message</span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"> &nbsp;<br/></span><span class="com">/* Latest data goes in the right most column, initialize it */</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> syscalls_by_latency </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[];</span><span class="pln"><br/></span><span class="kwd">for</span><span class="pln"> </span><span class="pun">(</span><span class="pln"> </span><span class="kwd">var</span><span class="pln"> index </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span><span class="pun">;</span><span class="pln"> index </span><span class="pun">&lt;</span><span class="pln"> </span><span class="lit">32</span><span class="pun">;</span><span class="pln"> index</span><span class="pun">++</span><span class="pln"> </span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/>syscalls_by_latency</span><span class="pun">[</span><span class="pln">index</span><span class="pun">]</span><span class="pln"> </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span><span class="pun">;</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="com">/* Presently we have the latency for each system call quantized in our message. Merge the data<br/>such that we have all the system call latency quantized together. This gives us the number<br/>of syscalls made with latencies in each particular band. */</span><span class="pln"><br/></span><span class="kwd">for</span><span class="pln"> </span><span class="pun">(</span><span class="pln"> </span><span class="kwd">var</span><span class="pln"> syscall </span><span class="kwd">in</span><span class="pln"> message </span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> val </span><span class="pun">=</span><span class="pln"> message</span><span class="pun">[</span><span class="pln">syscall</span><span class="pun">];</span><span class="pln"><br/></span><span class="kwd">for</span><span class="pln"> </span><span class="pun">(</span><span class="pln"> result_index </span><span class="kwd">in</span><span class="pln"> val </span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> latency_start </span><span class="pun">=</span><span class="pln"> val</span><span class="pun">[</span><span class="pln">result_index</span><span class="pun">][</span><span class="lit">0</span><span class="pun">][</span><span class="lit">0</span><span class="pun">];</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> count </span><span class="pun">=</span><span class="pln"> &nbsp;val</span><span class="pun">[</span><span class="pln">result_index</span><span class="pun">][</span><span class="lit">1</span><span class="pun">];</span><span class="pln"><br/></span><span class="com">/* The d script we\'re using lquantizes from 0 to 63 in steps of two. So dividing by 2 <br/>tells us which row this result belongs in */</span><span class="pln"><br/>syscalls_by_latency</span><span class="pun">[</span><span class="typ">Math</span><span class="pun">.</span><span class="pln">floor</span><span class="pun">(</span><span class="pln">latency_start</span><span class="pun">/</span><span class="lit">2</span><span class="pun">)]</span><span class="pln"> </span><span class="pun">+=</span><span class="pln"> count</span><span class="pun">;</span><span class="pln"> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="com">/* We just created a new column, shift the console to the left and add it. */</span><span class="pln"><br/>console_columns</span><span class="pun">.</span><span class="pln">shift</span><span class="pun">();</span><span class="pln"><br/>console_columns</span><span class="pun">.</span><span class="pln">push</span><span class="pun">(</span><span class="pln">syscalls_by_latency</span><span class="pun">);</span><span class="pln"><br/>drawArray</span><span class="pun">(</span><span class="pln">console_columns</span><span class="pun">);</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="com">/* Draw the columns and rows that map up the heatmap on to the canvas element */</span><span class="pln"><br/></span><span class="kwd">function</span><span class="pln"> drawArray</span><span class="pun">(</span><span class="pln">console_columns</span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> canvas </span><span class="pun">=</span><span class="pln"> document</span><span class="pun">.</span><span class="pln">getElementById</span><span class="pun">(</span><span class="str">\'canvas\'</span><span class="pun">);</span><span class="pln"><br/></span><span class="kwd">if</span><span class="pln"> </span><span class="pun">(</span><span class="pln">canvas</span><span class="pun">.</span><span class="pln">getContext</span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> ctx </span><span class="pun">=</span><span class="pln"> canvas</span><span class="pun">.</span><span class="pln">getContext</span><span class="pun">(</span><span class="str">\'2d\'</span><span class="pun">);</span><span class="pln"> &nbsp;<br/></span><span class="kwd">for</span><span class="pln"> </span><span class="pun">(</span><span class="pln"> </span><span class="kwd">var</span><span class="pln"> column_index </span><span class="kwd">in</span><span class="pln"> console_columns </span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> column </span><span class="pun">=</span><span class="pln"> console_columns</span><span class="pun">[</span><span class="pln">column_index</span><span class="pun">];</span><span class="pln"> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <br/></span><span class="kwd">for</span><span class="pln"> </span><span class="pun">(</span><span class="pln"> </span><span class="kwd">var</span><span class="pln"> entry_index </span><span class="kwd">in</span><span class="pln"> column </span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/>entry </span><span class="pun">=</span><span class="pln"> column</span><span class="pun">[</span><span class="pln">entry_index</span><span class="pun">];</span><span class="pln"><br/></span><span class="com">/* We\'re using a logarithmic scale for the brightness. This was all arrived at by<br/>trial and error and found to work well on my Mac. &nbsp;In the future this <br/>could all be adjustable with controls */</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> red_value </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span><span class="pun">;</span><span class="pln"> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<br/></span><span class="kwd">if</span><span class="pln"> </span><span class="pun">(</span><span class="pln"> entry </span><span class="pun">!=</span><span class="pln"> </span><span class="lit">0</span><span class="pln"> </span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/>red_value </span><span class="pun">=</span><span class="pln"> </span><span class="typ">Math</span><span class="pun">.</span><span class="pln">floor</span><span class="pun">(</span><span class="typ">Math</span><span class="pun">.</span><span class="pln">log</span><span class="pun">(</span><span class="pln">entry</span><span class="pun">)/</span><span class="typ">Math</span><span class="pun">.</span><span class="pln">log</span><span class="pun">(</span><span class="lit">2</span><span class="pun">));</span><span class="pln"> &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="com">//console.log(red_value); &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; </span><span class="pln"><br/>ctx</span><span class="pun">.</span><span class="pln">fillStyle </span><span class="pun">=</span><span class="pln"> </span><span class="str">\'rgb(\'</span><span class="pln"> </span><span class="pun">+</span><span class="pln"> </span><span class="pun">(</span><span class="pln">red_value </span><span class="pun">*</span><span class="pln"> </span><span class="lit">25</span><span class="pun">)</span><span class="pln"> </span><span class="pun">+</span><span class="pln"> </span><span class="str">\',0,0)\'</span><span class="pun">;</span><span class="pln"><br/>ctx</span><span class="pun">.</span><span class="pln">fillRect</span><span class="pun">(</span><span class="pln">column_index</span><span class="pun">*</span><span class="lit">16</span><span class="pun">,</span><span class="pln"> </span><span class="lit">496</span><span class="pun">-(</span><span class="pln">entry_index</span><span class="pun">*</span><span class="lit">16</span><span class="pun">),</span><span class="pln"> </span><span class="lit">16</span><span class="pun">,</span><span class="pln"> </span><span class="lit">16</span><span class="pun">);</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"> <br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="com">/* The heatmap is is really a 64x32 grid. Initialize the array which contains the grid data. */</span><span class="pln"><br/></span><span class="kwd">function</span><span class="pln"> setup</span><span class="pun">()</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/>console_columns </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[];</span><span class="pln"><br/></span><span class="kwd">for</span><span class="pln"> </span><span class="pun">(</span><span class="pln"> </span><span class="kwd">var</span><span class="pln"> column_index </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span><span class="pun">;</span><span class="pln"> column_index </span><span class="pun">&lt;</span><span class="pln"> </span><span class="lit">64</span><span class="pun">;</span><span class="pln"> column_index</span><span class="pun">++</span><span class="pln"> </span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"><br/></span><span class="kwd">var</span><span class="pln"> column </span><span class="pun">=</span><span class="pln"> </span><span class="pun">[];</span><span class="pln"><br/></span><span class="kwd">for</span><span class="pln"> </span><span class="pun">(</span><span class="pln"> </span><span class="kwd">var</span><span class="pln"> entry_index </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span><span class="pun">;</span><span class="pln"> entry_index </span><span class="pun">&lt;</span><span class="pln"> </span><span class="lit">32</span><span class="pun">;</span><span class="pln"> entry_index</span><span class="pun">++</span><span class="pln"> </span><span class="pun">)</span><span class="pln"> </span><span class="pun">{</span><span class="pln"> <br/>column</span><span class="pun">[</span><span class="pln">entry_index</span><span class="pun">]</span><span class="pln"> </span><span class="pun">=</span><span class="pln"> </span><span class="lit">0</span><span class="pun">;</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"><br/>console_columns</span><span class="pun">.</span><span class="pln">push</span><span class="pun">(</span><span class="pln">column</span><span class="pun">);</span><span class="pln"><br/></span><span class="pun">}</span><span class="pln"><br/></span><span class="pun">}</span></code></pre></div>\n\n<h2>Run It!</h2>\n\n<p>Run Heat Tacer with the following. Note, sudo is required by dtrace as it does kernal magic.</p>\n\n<pre><code><span class="pln">sudo node heat_tracer</span><span class="pun">.</span><span class="pln">js<br/></span></code></pre>\n\n<p>If all goes well you should see something a moving version of something like the image below.</p>\n\n<blockquote>\n  <p><img src="http://howtonode.org/heat-tracer/heat_tracer.png" alt="Alt value of image" title=""></img> </p>\n</blockquote>\n\n<h2>Contribute</h2>\n\n<p>You can find the latest version of Heat Tracer <a href="https://github.com/gflarity/Heat-Tracer">here</a>. It is my hope that this article will provide the ground work for a much more abitious performance analytics project. If you\'re interested in contributing please let me know.</p>\n\n<h2>Further Research</h2>\n\n<p>More information about Bryan and Brendan\'s demo can be found <a href="http://dtrace.org/blogs/brendan/2011/01/24/cloud-analytics-first-video/">here</a>.</p>\n\n<p>Socket.IO can be found <a href="http://socket.io/">here</a>.</p><hr></hr>\n\n<a href="http://disqus.com/forums/howtonodeorg/?url=ref">View the discussion thread.</a><a href="http://disqus.com" class="dsq-brlink">blog comments powered by</a>\n\n</div>' };
{'fraction_non_alphanumeric': 0.27427003103805037, 'fraction_numerical': 0.004052189906885849, 'mean_word_length': 8.111416601204503, 'pattern_counts': {'":': 0, '<': 4171, '<?xml version=': 0, '>': 4171, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4783569', 'n_tokens_mistral': 24010, 'n_tokens_neox': 22920, 'n_words': 4270}