<commit_before>var fs = require('fs'),
	request = require('request'),
	async = require('async'),
	RSS = require('rss'),
	rssOptions = require('./rssOptions');

var logFile = 'out/update.log';
var maxLogFileSize = 6000;

var items = [];

var getItems = function(ids, callback){
	async.each(ids, function(id, cb){
		console.log('requesting ' + id);
		request('http://zhuanlan.zhihu.com/api/columns/' + id + '/posts', function(error, response, body){
			body = JSON.parse(body);
			body.forEach(function(item){
				items.push(item);
			});
			console.log('got ' + body.length + ' from ' + id);
			cb();
		});
	}, function(error){
		if(!error){
			console.log("success got " + items.length + " items");
		}else{
			console.log("error : " + error);
		}
		if(callback){
			callback(error);
		}
	});
}

var sortItems = function(){
	items.sort(function(a, b){
		return new Date(b.publishedTime) - new Date(a.publishedTime);
	});
};

var saveToRss = function(path){
	if(items.length > 0){
		var pubDate = items[0].publishedTime;
		rssOptions.pubDate = pubDate;
		console.log('pubDate ' + pubDate);
		var feed = new RSS(rssOptions);
		var count = items.length;
		if(rssOptions.maxItemCount){
			count = Math.min(count, rssOptions.maxItemCount);
		}
		console.log('saving with ' + count + ' items');
		for(var i = 0; i < count; i++){
			feed.item(feedItemFromJson(items[i]));
		}
		fs.writeFile(path, feed.xml(), function (err) {
  			if (err) throw err;
  			var logStr = 'newest post time ' + pubDate +'\n';
  			if(fs.existsSync(logFile) && fs.statSync(logFile).size > maxLogFileSize){
  				fs.writeFileSync(logFile, logStr);
  			} else {
  				fs.appendFileSync(logFile, logStr);
  			}
		});
	}
}

var feedItemFromJson = function(json){
	var item = {
		title: json.title,
		description: json.content,
		url: 'http://zhuanlan.zhihu.com' + json.url,
		guid: json.slug,
		author: json.author.name + (json.author.bio ? ' | ' + json.author.bio : ''),
		date: json.publishedTime
	};
	return item;
}

module.exports = function(ids, path){
	getItems(ids, function(error){
		if(!error){
			sortItems();
			saveToRss(path);
		}
	});<commit_msg>test update.log diff<commit_after>var fs = require('fs'),
	request = require('request'),
	async = require('async'),
	RSS = require('rss'),
	rssOptions = require('./rssOptions');

var logFile = 'out/update.log';
var maxLogFileSize = 6000;

var items = [];

var getItems = function(ids, callback){
	async.each(ids, function(id, cb){
		console.log('requesting ' + id);
		request('http://zhuanlan.zhihu.com/api/columns/' + id + '/posts', function(error, response, body){
			body = JSON.parse(body);
			body.forEach(function(item){
				items.push(item);
			});
			console.log('got ' + body.length + ' from ' + id);
			cb();
		});
	}, function(error){
		if(!error){
			console.log("success got " + items.length + " items");
		}else{
			console.log("error : " + error);
		}
		if(callback){
			callback(error);
		}
	});
}

var sortItems = function(){
	items.sort(function(a, b){
		return new Date(b.publishedTime) - new Date(a.publishedTime);
	});
};

var saveToRss = function(path){
	if(items.length > 0){
		var pubDate = items[0].publishedTime;
		rssOptions.pubDate = pubDate;
		console.log('pubDate ' + pubDate);
		var feed = new RSS(rssOptions);
		var count = items.length;
		if(rssOptions.maxItemCount){
			count = Math.min(count, rssOptions.maxItemCount);
		}
		console.log('saving with ' + count + ' items');
		for(var i = 0; i < count; i++){
			feed.item(feedItemFromJson(items[i]));
		}
		fs.writeFile(path, feed.xml(), function (err) {
  			if (err) throw err;
  			var logStr = 'newest post ' + pubDate +'\n';
  			if(fs.existsSync(logFile) && fs.statSync(logFile).size > maxLogFileSize){
  				fs.writeFileSync(logFile, logStr);
  			} else {
  				fs.appendFileSync(logFile, logStr);
  			}
		});
	}
}

var feedItemFromJson = function(json){
	var item = {
		title: json.title,
		description: json.content,
		url: 'http://zhuanlan.zhihu.com' + json.url,
		guid: json.slug,
		author: json.author.name + (json.author.bio ? ' | ' + json.author.bio : ''),
		date: json.publishedTime
	};
	return item;
}

module.exports = function(ids, path){
	getItems(ids, function(error){
		if(!error){
			sortItems();
			saveToRss(path);
		}
	});
{'fraction_non_alphanumeric': 0.18230877522290004, 'fraction_numerical': 0.00328484279680901, 'mean_word_length': 3.68976897689769, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3332592', 'n_tokens_mistral': 1696, 'n_tokens_neox': 1553, 'n_words': 303}