<commit_before><html>
  <head>
    <!--<script src="/static/js/processing.js"></script>-->
    <script src="/static/js/jquery.js"></script>
    <!--<script src="/static/js/socket.io.js"></script>-->
    <script type="text/javascript">
      var C=100;

      var canvas;
      var ctx;

      var pmouseX=null;
      var pmouseY=null;

      var portwidth=0.016102;
      var portheight=0.0106;

      var ll=-71.116102;
      var rr=ll+portwidth;
      var bb=42.3594;
      var tt=bb+portheight;

      var tilewidth=0.02; // geographical units
      var tiles={};

      var unitscale=1000;
      var lineproportion=0.00031;

      var dijkstra;
      var graph = new Graph();
      var tree;
  
      function clear(){
        ctx.clearRect(ll,bb,(rr-ll),(tt-bb));
      }

      function edge_weight(prev_edge, edge) {
        if( edge.keysvals["oneway:bicycle"]==="yes" && edge.reverse ){
          return null;
        }

        var mult=1;
        if( edge.keysvals.highway === "motorway" || edge.keysvals.highway === "motorway_link"){
          return null;
        }
        if( edge.keysvals.highway === "footway" || edge.keysvals.highway === "cycleway" ) {
          mult = 1.0//;0.75;
        } else if( edge.keysvals.highway === "primary" ) {
          mult = 1.0//;1.5;
        } else if( edge.keysvals.bicycle === "designated" ) {
          mult = 0.7;
        } else if( edge.keysvals.highway === "steps" ) {
          mult = 10;
        }

        var ret = 0;
        if(edge.loc.length<2){
          return 0;
        }
        for(var i=0; i<edge.loc.length-1; i++) {
          ret += dist( edge.loc[i][0], edge.loc[i][1], edge.loc[i+1][0], edge.loc[i+1][1] );
        }

        var corner_penalty=0.0;
        if(prev_edge && prev_edge.keysvals.name!==edge.keysvals.name){
          corner_penalty=0.00005;
        }

        return mult*ret + corner_penalty;
      }

      function draw_linestring(ctx, ary){
        ctx.beginPath();
        ctx.moveTo(ary[0][0]*unitscale,ary[0][1]*unitscale);
        for(var j=1; j<ary.length; j++){
          if(!ary[j][0] || !ary[j][1])
            continue;
          ctx.lineTo(ary[j][0]*unitscale, ary[j][1]*unitscale);
        }
        ctx.stroke();
      }

      function draw_way(ctx, way, linewidth){
        if( linewidth === undefined ) {
          ctx.lineWidth=0.000005*Math.pow((rr-ll),0.5)*10*unitscale;
        } else {
          ctx.lineWidth=linewidth*unitscale;
        }

        if(way.loc.length<2)
          return;
        if(!way.loc[0][0] || !way.loc[0][1])
          return;

        draw_linestring( ctx, way.loc );
      }

      function Dijkstra(startnode){
        this.startnode = startnode;
        this.queue = new Queue();
        this.running = null;

        //dummy edge goes to origin
        this.queue.insert( {'orig':null,'dest':startnode,'way':null,'edgeweight':0}, startnode, 0 );
        
        this.step = function(exponent){  
          if( this.queue.empty() ){
            return;
          }

          // get edge to nearest unreached vertex
          var best_edge = this.queue.extract_min();
          var best_edge_orig=best_edge[0]['orig']
          var best_edge_dest=best_edge[0]['dest'];
          ctx.strokeStyle="red";
          var best_edge_weight=best_edge[1];

          // stick that edge in the tree (destination points back to origin)
          tree[best_edge_dest]={'orig':best_edge_orig,'weight':best_edge_weight,'edgeweight':best_edge[0]['edgeweight'],'trunkyness':0,'way':best_edge[0]['way']};

          // draw the edge
          draw_tree_branch( ctx, tree[best_edge_dest], exponent );
          //trace up the tree adding the edge weight
          var parentid=best_edge_orig;
          while(parentid){
            tree[parentid]['trunkyness']+=best_edge[0]['edgeweight'];
            draw_tree_branch( ctx, tree[parentid], exponent );
            parentid = tree[parentid]['orig']
          }

          // for each outgoing edge
          var outgoing = graph.adj[best_edge_dest];
          if(!outgoing)
            return;

          for( var i=0; i<outgoing.length; i++ ){
            var candidate_edge = outgoing[i];
            var cand_dest_node = candidate_edge[0];
            var cand_edge_data = candidate_edge[1];
            var cand_edge_weight = edge_weight( best_edge[0].way, cand_edge_data );
            if(cand_edge_weight===null){
              continue;
            }
            if( tree[cand_dest_node] !== undefined ){
              continue;
            }

            // insert into queue
            this.queue.insert( {'orig':best_edge_dest, 'dest':cand_dest_node,'way':cand_edge_data, 'edgeweight':cand_edge_weight}, cand_dest_node, best_edge_weight+cand_edge_weight );
          }
        }

        this.run = function(exponent){
          this.running=true;
          var metathis=this;
          var metastep = function(){
            if(!metathis.running){
              return;
            }            

            if(metathis.queue.empty())
              return;
            for(var i=0; i<10; i++){
             metathis.step(exponent);
            }
            setTimeout(metastep, 0);
          }
          metastep();
        }

        this.stop = function(){
          this.running=false;
        }
      }

      function interpolate( ary, spacing ) {
        if(ary.length<2){
          return ary;
        }

        var ret = [];
        
        var cur=0;
        var segstart=0;
        var segend;
        for(var i=1; i<ary.length; i++) {
          var segdist = dist(ary[i-1][0],ary[i-1][1],ary[i][0],ary[i][1]);
          var norm = [(ary[i][0]-ary[i-1][0])/segdist, (ary[i][1]-ary[i-1][1])/segdist];

          segend = segstart + segdist;
          while(cur<segend){
            var seginterp = cur-segstart;
            var yld = [norm[0]*seginterp+ary[i-1][0], norm[1]*seginterp+ary[i-1][1]]
            ret.push( yld );

            cur += spacing;
          }
          ret.push(ary[i]);
          segstart = segend;
        }

        return ret;
      }

      function reverse(ary){
        ret = [];
        for(var i=ary.length-1; i--; i>=0){
          ret.push( ary[i] );
        }
        return ret;
      }

      function draw_tree_branch(ctx,branch, exponent){
          if( exponent === undefined ){ exponent = 0.35; }

          if( branch['way'] ){

            if(branch.way.keysvals.highway==="footway" || branch.way.keysvals.highway==="path") {
              ctx.strokeStyle="green";
            } else if (branch.way.keysvals.bicycle==="designated") {
              ctx.strokeStyle="purple"; 
            } else if (branch.way.keysvals.highway==="cycleway") {
              ctx.strokeStyle="blue";
            } else {
              ctx.strokeStyle="red";
            }
            
            if(branch['trunkyness']==0){
              var segs = interpolate( branch.way.loc, 0.0001);
              var len = 0;
              if( branch.way.reverse ){
                for(var i=1; i<segs.length; i++){
                  len += dist(segs[i-1][0],segs[i-1][1],segs[i][0],segs[i][1]);
                  ctx.lineWidth=Math.pow(len,exponent)*0.0005*unitscale*Math.pow((rr-ll),0.5)*5;
                  draw_linestring( ctx, [segs[i-1],segs[i]] );
                }
              } else {
                for(var i=segs.length-1; i>=1; i--){
                  len += dist(segs[i-1][0],segs[i-1][1],segs[i][0],segs[i][1]);
                  ctx.lineWidth=Math.pow(len,exponent)*0.0005*unitscale*Math.pow((rr-ll),0.5)*5;
                  draw_linestring( ctx, [segs[i-1],segs[i]] );
                }
              }
            } else {
              var branchwidth=Math.pow(branch['trunkyness'],exponent)*0.0005*Math.pow((rr-ll),0.5)*5;
              draw_way(ctx,branch['way'],branchwidth);
            }
       
          }
      }

      function draw_tree(ctx,tree){
        for(var item in tree){
          draw_tree_branch(ctx,tree[item]);
        }
      }

      function Graph(){
        this.adj = {}; //nodeid -> list of (nodeid, edge)
        
        this.add = function(fromv,tov,edge){
          if(this.adj[fromv]===undefined){
            this.adj[fromv]=[];
          }
          this.adj[fromv].push( [tov, edge] );
        }

      }

      function Queue(){
        this.data = {};
        
        this.insert = function(item,label,prio){
          if( this.data[label] !== undefined ) {
            if(this.data[label][1]>prio){
              this.data[label] = [item,prio];
            }
          } else {
            this.data[label] = [item,prio];
          }
        } 

        this.min = function() {
          var min=null;
          var minweight=10000000;
          for( var item in this.data ){
            if ( this.data[item][1] < minweight ) {
              minweight = this.data[item][1];
              min=item; 
            }
          }
          return min;
        }

        this.extract_min = function() {
          var minkey = this.min();
          var ret = this.data[minkey];
          delete this.data[minkey];
          return ret;
        }

        this.empty = function() {
          for( var item in this.data ){
            return false;
          }
          return true;
        }
      }
      

      function Tile(ll,bb,ww,hh){
        this.ll=ll;
        this.bb=bb;
        this.ww=ww;
        this.hh=hh;

        this.ways = []

        this.nearestNode = function(x,y){
          var winnerdist=100000000;
          var winner=null;
          for(var i=0; i<this.ways.length; i++){
            var way = this.ways[i];

            if(!way.loc || way.loc.length==0){
              continue;
            }
            var s0 = way.loc[0];
            var dists0 = dist(s0[0],s0[1],x,y);
            if(dists0<winnerdist){
              winnerdist = dists0;
              winner=way.nodes[0];
            }
          
            var s1 = way.loc[way.loc.length-1];
            var dists1 = dist(s1[0],s1[1],x,y);
            if(dists1<winnerdist){
              winnerdist = dists1
              winner=way.nodes[way.nodes.length-1];
            }
          }
          return winner;
        }

        this.draw = function(ctx, drawways){

          if(drawways!==false){
            for(var i=0; i<this.ways.length; i++){
              this.drawWay(ctx,i); 
            }
          }
        }

        this.drawWay = function(ctx,i){

              var way = this.ways[i];
              draw_way( ctx, way );
        }

        this.tilekey = function(){
          return this.ll.toFixed(2)+":"+this.bb.toFixed(2);
        }

        this.fetch = function(ctx, graph){
          var metathis=this;
          $.getJSON("/tile/"+this.tilekey(), function(data){
            if(!data)
              return;
            metathis.ways = data.value.ways;
 
            metathis.draw(ctx);

            for(var i=0; i<data.value.ways.length; i++){
              var way = data.value.ways[i];
              if( way.loc.length<2 || way.nodes.length<2 ){
                 continue;
              }
              graph.add( way.nodes[0], way.nodes[way.nodes.length-1], way );
              if( way.keysvals.oneway !== "true" && way.keysvals.oneway !== "yes" ){
                var revway = {reverse:true,loc:way.loc,keysvals:way.keysvals,id:way.id,nodes:way.nodes}
                graph.add( way.nodes[way.nodes.length-1], way.nodes[0], revway );
              }
            }
          });

          
        }

      }

      function dist(x1,y1,x2,y2) {
        return Math.sqrt( Math.pow(x1-x2,2)+Math.pow(y1-y2,2) );
      }
      function min(a,b){
        if(a>b) {
          return a;
        } else {
          return b;
        }
      }
      function mag(x,y){
        return Math.sqrt( Math.pow(x,2)+Math.pow(y,2) );
      }

      function apply_transform(){
        ctx.setTransform(1,0,0,1,0,0);

        var xscale = canvas.width/((rr-ll)*unitscale);
        var yscale = canvas.height/((bb-tt)*unitscale);

        ctx.translate( -ll*xscale*unitscale, -tt*yscale*unitscale );
        ctx.scale(xscale,yscale);
      }
      function redraw() {
        ctx.clearRect(ll*unitscale,bb*unitscale,(rr-ll)*unitscale,(tt-bb)*unitscale);
        ctx.strokeStyle="red";
        draw_tree(ctx,tree);
        ctx.strokeStyle="black";

        for(tile in tiles){
          tiles[tile].draw(ctx);
        }
      }

      function fetchTiles() {

        if((rr-ll)/tilewidth > 20 || (tt-bb)/tilewidth > 20)
          return; //too big
<commit_msg>less garish color choices<commit_after><html>
  <head>
    <!--<script src="/static/js/processing.js"></script>-->
    <script src="/static/js/jquery.js"></script>
    <!--<script src="/static/js/socket.io.js"></script>-->
    <script type="text/javascript">
      var C=100;

      var canvas;
      var ctx;

      var pmouseX=null;
      var pmouseY=null;

      var portwidth=0.016102;
      var portheight=0.0106;

      var ll=-71.116102;
      var rr=ll+portwidth;
      var bb=42.3594;
      var tt=bb+portheight;

      var tilewidth=0.02; // geographical units
      var tiles={};

      var unitscale=1000;
      var lineproportion=0.00031;

      var dijkstra;
      var graph = new Graph();
      var tree;
  
      function clear(){
        ctx.clearRect(ll,bb,(rr-ll),(tt-bb));
      }

      function edge_weight(prev_edge, edge) {
        if( edge.keysvals["oneway:bicycle"]==="yes" && edge.reverse ){
          return null;
        }

        var mult=1;
        if( edge.keysvals.highway === "motorway" || edge.keysvals.highway === "motorway_link"){
          return null;
        }
        if( edge.keysvals.highway === "footway" || edge.keysvals.highway === "cycleway" ) {
          mult = 1.0//;0.75;
        } else if( edge.keysvals.highway === "primary" ) {
          mult = 1.0//;1.5;
        } else if( edge.keysvals.bicycle === "designated" ) {
          mult = 0.7;
        } else if( edge.keysvals.highway === "steps" ) {
          mult = 10;
        }

        var ret = 0;
        if(edge.loc.length<2){
          return 0;
        }
        for(var i=0; i<edge.loc.length-1; i++) {
          ret += dist( edge.loc[i][0], edge.loc[i][1], edge.loc[i+1][0], edge.loc[i+1][1] );
        }

        var corner_penalty=0.0;
        if(prev_edge && prev_edge.keysvals.name!==edge.keysvals.name){
          corner_penalty=0.00005;
        }

        return mult*ret + corner_penalty;
      }

      function draw_linestring(ctx, ary){
        ctx.beginPath();
        ctx.moveTo(ary[0][0]*unitscale,ary[0][1]*unitscale);
        for(var j=1; j<ary.length; j++){
          if(!ary[j][0] || !ary[j][1])
            continue;
          ctx.lineTo(ary[j][0]*unitscale, ary[j][1]*unitscale);
        }
        ctx.stroke();
      }

      function draw_way(ctx, way, linewidth){
        if( linewidth === undefined ) {
          ctx.lineWidth=0.000005*Math.pow((rr-ll),0.5)*10*unitscale;
        } else {
          ctx.lineWidth=linewidth*unitscale;
        }

        if(way.loc.length<2)
          return;
        if(!way.loc[0][0] || !way.loc[0][1])
          return;

        draw_linestring( ctx, way.loc );
      }

      function Dijkstra(startnode){
        this.startnode = startnode;
        this.queue = new Queue();
        this.running = null;

        //dummy edge goes to origin
        this.queue.insert( {'orig':null,'dest':startnode,'way':null,'edgeweight':0}, startnode, 0 );
        
        this.step = function(exponent){  
          if( this.queue.empty() ){
            return;
          }

          // get edge to nearest unreached vertex
          var best_edge = this.queue.extract_min();
          var best_edge_orig=best_edge[0]['orig']
          var best_edge_dest=best_edge[0]['dest'];
          ctx.strokeStyle="D40000";
          var best_edge_weight=best_edge[1];

          // stick that edge in the tree (destination points back to origin)
          tree[best_edge_dest]={'orig':best_edge_orig,'weight':best_edge_weight,'edgeweight':best_edge[0]['edgeweight'],'trunkyness':0,'way':best_edge[0]['way']};

          // draw the edge
          draw_tree_branch( ctx, tree[best_edge_dest], exponent );
          //trace up the tree adding the edge weight
          var parentid=best_edge_orig;
          while(parentid){
            tree[parentid]['trunkyness']+=best_edge[0]['edgeweight'];
            draw_tree_branch( ctx, tree[parentid], exponent );
            parentid = tree[parentid]['orig']
          }

          // for each outgoing edge
          var outgoing = graph.adj[best_edge_dest];
          if(!outgoing)
            return;

          for( var i=0; i<outgoing.length; i++ ){
            var candidate_edge = outgoing[i];
            var cand_dest_node = candidate_edge[0];
            var cand_edge_data = candidate_edge[1];
            var cand_edge_weight = edge_weight( best_edge[0].way, cand_edge_data );
            if(cand_edge_weight===null){
              continue;
            }
            if( tree[cand_dest_node] !== undefined ){
              continue;
            }

            // insert into queue
            this.queue.insert( {'orig':best_edge_dest, 'dest':cand_dest_node,'way':cand_edge_data, 'edgeweight':cand_edge_weight}, cand_dest_node, best_edge_weight+cand_edge_weight );
          }
        }

        this.run = function(exponent){
          this.running=true;
          var metathis=this;
          var metastep = function(){
            if(!metathis.running){
              return;
            }            

            if(metathis.queue.empty())
              return;
            for(var i=0; i<10; i++){
             metathis.step(exponent);
            }
            setTimeout(metastep, 0);
          }
          metastep();
        }

        this.stop = function(){
          this.running=false;
        }
      }

      function interpolate( ary, spacing ) {
        if(ary.length<2){
          return ary;
        }

        var ret = [];
        
        var cur=0;
        var segstart=0;
        var segend;
        for(var i=1; i<ary.length; i++) {
          var segdist = dist(ary[i-1][0],ary[i-1][1],ary[i][0],ary[i][1]);
          var norm = [(ary[i][0]-ary[i-1][0])/segdist, (ary[i][1]-ary[i-1][1])/segdist];

          segend = segstart + segdist;
          while(cur<segend){
            var seginterp = cur-segstart;
            var yld = [norm[0]*seginterp+ary[i-1][0], norm[1]*seginterp+ary[i-1][1]]
            ret.push( yld );

            cur += spacing;
          }
          ret.push(ary[i]);
          segstart = segend;
        }

        return ret;
      }

      function reverse(ary){
        ret = [];
        for(var i=ary.length-1; i--; i>=0){
          ret.push( ary[i] );
        }
        return ret;
      }

      function draw_tree_branch(ctx,branch, exponent){
          if( exponent === undefined ){ exponent = 0.35; }

          if( branch['way'] ){

            if(branch.way.keysvals.highway==="footway" || branch.way.keysvals.highway==="path") {
              ctx.strokeStyle="007007";
            } else if (branch.way.keysvals.bicycle==="designated") {
              ctx.strokeStyle="910088"; 
            } else if (branch.way.keysvals.highway==="cycleway") {
              ctx.strokeStyle="000BA8";
            } else {
              ctx.strokeStyle="D40000";
            }
            
            if(branch['trunkyness']==0){
              var segs = interpolate( branch.way.loc, 0.0001);
              var len = 0;
              if( branch.way.reverse ){
                for(var i=1; i<segs.length; i++){
                  len += dist(segs[i-1][0],segs[i-1][1],segs[i][0],segs[i][1]);
                  ctx.lineWidth=Math.pow(len,exponent)*0.0005*unitscale*Math.pow((rr-ll),0.5)*5;
                  draw_linestring( ctx, [segs[i-1],segs[i]] );
                }
              } else {
                for(var i=segs.length-1; i>=1; i--){
                  len += dist(segs[i-1][0],segs[i-1][1],segs[i][0],segs[i][1]);
                  ctx.lineWidth=Math.pow(len,exponent)*0.0005*unitscale*Math.pow((rr-ll),0.5)*5;
                  draw_linestring( ctx, [segs[i-1],segs[i]] );
                }
              }
            } else {
              var branchwidth=Math.pow(branch['trunkyness'],exponent)*0.0005*Math.pow((rr-ll),0.5)*5;
              draw_way(ctx,branch['way'],branchwidth);
            }
       
          }
      }

      function draw_tree(ctx,tree){
        for(var item in tree){
          draw_tree_branch(ctx,tree[item]);
        }
      }

      function Graph(){
        this.adj = {}; //nodeid -> list of (nodeid, edge)
        
        this.add = function(fromv,tov,edge){
          if(this.adj[fromv]===undefined){
            this.adj[fromv]=[];
          }
          this.adj[fromv].push( [tov, edge] );
        }

      }

      function Queue(){
        this.data = {};
        
        this.insert = function(item,label,prio){
          if( this.data[label] !== undefined ) {
            if(this.data[label][1]>prio){
              this.data[label] = [item,prio];
            }
          } else {
            this.data[label] = [item,prio];
          }
        } 

        this.min = function() {
          var min=null;
          var minweight=10000000;
          for( var item in this.data ){
            if ( this.data[item][1] < minweight ) {
              minweight = this.data[item][1];
              min=item; 
            }
          }
          return min;
        }

        this.extract_min = function() {
          var minkey = this.min();
          var ret = this.data[minkey];
          delete this.data[minkey];
          return ret;
        }

        this.empty = function() {
          for( var item in this.data ){
            return false;
          }
          return true;
        }
      }
      

      function Tile(ll,bb,ww,hh){
        this.ll=ll;
        this.bb=bb;
        this.ww=ww;
        this.hh=hh;

        this.ways = []

        this.nearestNode = function(x,y){
          var winnerdist=100000000;
          var winner=null;
          for(var i=0; i<this.ways.length; i++){
            var way = this.ways[i];

            if(!way.loc || way.loc.length==0){
              continue;
            }
            var s0 = way.loc[0];
            var dists0 = dist(s0[0],s0[1],x,y);
            if(dists0<winnerdist){
              winnerdist = dists0;
              winner=way.nodes[0];
            }
          
            var s1 = way.loc[way.loc.length-1];
            var dists1 = dist(s1[0],s1[1],x,y);
            if(dists1<winnerdist){
              winnerdist = dists1
              winner=way.nodes[way.nodes.length-1];
            }
          }
          return winner;
        }

        this.draw = function(ctx, drawways){

          if(drawways!==false){
            for(var i=0; i<this.ways.length; i++){
              this.drawWay(ctx,i); 
            }
          }
        }

        this.drawWay = function(ctx,i){

              var way = this.ways[i];
              draw_way( ctx, way );
        }

        this.tilekey = function(){
          return this.ll.toFixed(2)+":"+this.bb.toFixed(2);
        }

        this.fetch = function(ctx, graph){
          var metathis=this;
          $.getJSON("/tile/"+this.tilekey(), function(data){
            if(!data)
              return;
            metathis.ways = data.value.ways;
 
            metathis.draw(ctx);

            for(var i=0; i<data.value.ways.length; i++){
              var way = data.value.ways[i];
              if( way.loc.length<2 || way.nodes.length<2 ){
                 continue;
              }
              graph.add( way.nodes[0], way.nodes[way.nodes.length-1], way );
              if( way.keysvals.oneway !== "true" && way.keysvals.oneway !== "yes" ){
                var revway = {reverse:true,loc:way.loc,keysvals:way.keysvals,id:way.id,nodes:way.nodes}
                graph.add( way.nodes[way.nodes.length-1], way.nodes[0], revway );
              }
            }
          });

          
        }

      }

      function dist(x1,y1,x2,y2) {
        return Math.sqrt( Math.pow(x1-x2,2)+Math.pow(y1-y2,2) );
      }
      function min(a,b){
        if(a>b) {
          return a;
        } else {
          return b;
        }
      }
      function mag(x,y){
        return Math.sqrt( Math.pow(x,2)+Math.pow(y,2) );
      }

      function apply_transform(){
        ctx.setTransform(1,0,0,1,0,0);

        var xscale = canvas.width/((rr-ll)*unitscale);
        var yscale = canvas.height/((bb-tt)*unitscale);

        ctx.translate( -ll*xscale*unitscale, -tt*yscale*unitscale );
        ctx.scale(xscale,yscale);
      }
      function redraw() {
        ctx.clearRect(ll*unitscale,bb*unitscale,(rr-ll)*unitscale,(tt-bb)*unitscale);
        ctx.strokeStyle="D40000";
        draw_tree(ctx,tree);
        ctx.strokeStyle="black";

        for(tile in tiles){
          tiles[tile].draw(ctx);
        }
      }

      function fetchTiles() {

        if((rr-ll)/tilewidth > 20 || (tt-bb)/tilewidth > 20)
          return; //too big

{'fraction_non_alphanumeric': 0.15318402805451503, 'fraction_numerical': 0.022435641986132144, 'mean_word_length': 1.5967508278145695, 'pattern_counts': {'":': 2, '<': 61, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6232109', 'n_tokens_mistral': 8399, 'n_tokens_neox': 8038, 'n_words': 1289}