<commit_before>"use strict";

const api = require('./api2');
const http = require('http');
const zlib = require('zlib');
const Url = require('url');

const nodes = api.config.nodes;
exports.nodes=nodes;

if(api.config.update){
api.update.on("update",function(file,stamp,id){
	if(stamp<Date.now()-24*60*60)return;
	const s="/"+file+"/"+stamp+"/"+id+"/:"+api.port+"+server.cgi";
    for(var n of api.config.friends){
        get(nodeUrl(n,"update")+s);
     }
});
}

function update(file,stamp,id,node){
	console.log("update:"+file+" "+stamp+" "+id+" "+node);
    api.spam(id).then(function(){
	api.thread.get(file,{time:stamp,id:id,head:true}).then(function(rows){
		if(rows.length!=0)return;
		readLine(nodeUrl(node,"get",file)+"/"+stamp+"/"+id,function(body){
			const x = body.match(/(\d+)<>(.{32})<>(.*)/);
			if(x&&stamp==x[1]&&id==x[2])api.thread.post(file,stamp|0,id,x[3]);
		});
	});
    });
}
function readAll(node,file){
    api.threads.create(api.getTitle(file),function() {
    readLine(nodeUrl(node,"get",file,api.config.range.newThread),function(body){
        const x = body.match(/(\d+)<>(.{32})<>(.*)/);
        api.thread.post(file,x[1]|0,x[2],x[3]);
    });
	});
}
function readHead(node,file){
    const t = api.config.range.head;
	api.thread.get(file,{time:time(t),head:true}).then(function(rows){
		var list=new Array(rows.length);
		for(var i=0;i<rows.length;i++){
			list[i]=rows[i].stamp+"<>"+rows[i].id;
		}
        var num=0;
        const len=list.length;
        var next=[];
		readLine(nodeUrl(node,"head",file,t),function(body){
            var i=list.indexOf(body);
			if(i==-1){
                next.push(body);
			}else{
                list[i]=undefined;
                num++;
            }
		},function(){/*
            const per=next.length;
            if(per==0)return;
            const c=(per>30)||(len!=0)&&((num*3<len)||(per>len*30));
            //len 0:30 10:10 20:20
            if(false){
                console.log("immunity:"+len+" "+per+" "+num+" in "+node+" at "+file);
                return;
            }
            // when new>0 and old>0 , it must be new:old<=1:3 match:old>=1:2
            //limit new<=30*/
            for(var i=0;i<next.length;i++){
                var x = next[i].split("<>");
                update(file,x[0],x[1],node);
            }
        });
	});
}
function readNode(node,t){
    if(!node)return;
	api.threads.get({}).then(function(rows){
        t = t||api.config.range.node;
        var done =[];
		readLine(nodeUrl(node,"recent","",t),function(body){
			const x = body.split("<>");
			for(var i=0;i<rows.length;i++){
				if(rows[i]&&rows[i].file==x[2]){
                    if(done.indexOf(x[2])!=-1)return;
                    done.push(x[2]);
					if(!(rows[i].laststamp==x[0]&&rows[i].lastid==x[1]))readHead(node,x[2]);
					return;<commit_msg>v0.46<commit_after>"use strict";

const api = require('./api2');
const http = require('http');
const zlib = require('zlib');
const Url = require('url');

const nodes = api.config.nodes;
exports.nodes=nodes;

if(api.config.update){
api.update.on("update",function(file,stamp,id){
	if(stamp<Date.now()-24*60*60)return;
	const s="/"+file+"/"+stamp+"/"+id+"/:"+api.port+"+server.cgi";
    for(var n of api.config.friends){
        get(nodeUrl(n,"update")+s);
     }
});
}

function update(file,stamp,id,node){
	console.log("update:"+file+" "+stamp+" "+id+" "+node);
    api.spam(id).then(function(){
	api.thread.get(file,{time:stamp,id:id,head:true}).then(function(rows){
		if(rows.length!=0)return;
		readLine(nodeUrl(node,"get",file)+"/"+stamp+"/"+id,function(body){
			const x = body.match(/(\d+)<>(.{32})<>(.*)/);
			if(x&&stamp==x[1]&&id==x[2])api.thread.post(file,stamp|0,id,x[3]);
		});
	});
    });
}
function readAll(node,file){
    api.threads.create(api.getTitle(file),function() {
    readLine(nodeUrl(node,"get",file,api.config.range.newThread),function(body){
        const x = body.match(/(\d+)<>(.{32})<>(.*)/);
        api.thread.post(file,x[1]|0,x[2],x[3]);
    });
	});
}
function readHead(node,file){
    const t = api.config.range.head;
	api.thread.get(file,{time:time(t),head:true}).then(function(rows){
		var list=new Array(rows.length);
		for(var i=0;i<rows.length;i++){
			list[i]=rows[i].stamp+"<>"+rows[i].id;
		}
        var num=0;
        const len=list.length;
        var next=[];
		readLine(nodeUrl(node,"head",file,t),function(body){
            var i=list.indexOf(body);
			if(i==-1){
                next.push(body);
			}else{
                list[i]=undefined;
                num++;
            }
		},function(){/*
            const per=next.length;
            if(per==0)return;
            const c=(per>30)||(len!=0)&&((num*3<len)||(per>len*30));
            //len 0:30 10:10 20:20
            if(false){
                console.log("immunity:"+len+" "+per+" "+num+" in "+node+" at "+file);
                return;
            }
            // when new>0 and old>0 , it must be new:old<=1:3 match:old>=1:2
            //limit new<=30*/
            for(var i=0;i<next.length;i++){
                var x = next[i].split("<>");
                api.spam(x[1]).then(function() {
					update(file,x[0],x[1],node);
				},function() {
					
				});
            }
        });
	});
}
function readNode(node,t){
    if(!node)return;
	api.threads.get({}).then(function(rows){
        t = t||api.config.range.node;
        var done =[];
		readLine(nodeUrl(node,"recent","",t),function(body){
			const x = body.split("<>");
			for(var i=0;i<rows.length;i++){
				if(rows[i]&&rows[i].file==x[2]){
                    if(done.indexOf(x[2])!=-1)return;
                    done.push(x[2]);
					if(!(rows[i].laststamp==x[0]&&rows[i].lastid==x[1]))readHead(node,x[2]);
					return;
{'fraction_non_alphanumeric': 0.2412349555206698, 'fraction_numerical': 0.021629164486307344, 'mean_word_length': 2.8900949796472184, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6365473', 'n_tokens_mistral': 2294, 'n_tokens_neox': 2150, 'n_words': 263}