<commit_before>var util = require('util'),
	request = require('request'),
	hp = require("htmlparser"),
	PageCrawlingStream = require('./stream/page-crawling-stream'),
	map = require('./map');

var jar = request.jar(),
	cookie = request.cookie("over18=1"),
	request = request.defaults({jar: jar});

var _text = function (node) {
	var buffer = [], str;
	_text0(buffer, node);
	str = buffer.join('');
	return str[str.length - 1] === '\n' ? str : str + '\n';
};

var _text0 = function (buffer, node) {
	if (node.type === 'text')
		buffer.push(node.data);
	else if (node.children)
		node.children.forEach(function (c) { _text0(buffer, c); });
};

var _parseComment = function (node) {
	var subtype, author, content, time;
	if (!node.children)
		return {};
	node.children.forEach(function (c) {
		var cls = c.attribs.class,
			cnt = (c.children && c.children[0] && c.children[0].data) || '';
		if (!cls)
			return;
		if (cls.indexOf('push-tag') > -1)
			subtype = 
				cnt.indexOf('推') > -1 ? 1 : 
				cnt.indexOf('噓') > -1 ? 2 : 3;
		else if (cls.indexOf('push-userid') > -1)
			author = cnt;
		else if (cls.indexOf('push-content') > -1)
			content = cnt.substring(1);
		else if (cls.indexOf('push-ipdatetime') > -1)
			time = cnt.trim();
	});
	return { type: 'comment', subtype: subtype, author: author, 
		content: content, time: time };
};

var _parseMetaline = function (node) {
	var tag, value;
	if (!node.children)
		return {};
	node.children.forEach(function (c) {
		var cls = c.attribs.class,
			cnt = (c.children && c.children[0] && c.children[0].data) || '';
		if (!cls)
			return;
		if (cls.indexOf('article-meta-tag') > -1)
			tag = cnt;
		else if (cls.indexOf('article-meta-value') > -1)
			value = cnt;
	});
	return { tag: tag, value: value };
};

var _parsePostResponse = function (data, raw) {
	var url = data.url,
		body = data.body,
		handler, parser;
	
	handler = new hp.DefaultHandler(function (error, dom) {}, { 
		verbose: false, ignoreWhitespace: true
	});
	parser = new hp.Parser(handler);
	parser.parseComplete(body);
	
	// title
	// <meta property="og:title" content="[TITLE]" />
	var tm = /property="og:title" content="([^"]*)"/.exec(body), title;
	if (tm)
		title = tm[1];
	
	// list of content/comments
	var content = [], buffer = [], meta = [];
	hp.DomUtils.getElementById('main-content', handler.dom).children.forEach(function (node) {
		var cls = node.attribs && node.attribs.class;
		
		if (cls && cls.indexOf('article-metaline') > -1) {
			meta.push(_parseMetaline(node));
			return;
		}
		
		if (cls && cls.indexOf('richcontent') > -1)
			return;
		
		if (cls && cls.indexOf('push') > -1) {
			if (buffer.length)
				content.push(buffer.join(''));
			buffer = [];
			content.push(_parseComment(node));
			return;
		}
		
		buffer.push(_text(node));
	});
	if (buffer.length)
		content.push(buffer.join(''));
	
	var result = {
		board: url.board,
		post: url.post,
		title: title,
		meta: meta,
		content: content
	};
	
	if (raw)
		result.raw = data.body;
	
	return result;
};

var util = {
	
	getBoardUrl: function (board, pageNum) {
		var url = "http://www.ptt.cc/bbs/" + board + 
			"/index" + (pageNum != null ? pageNum : "") + ".html"
		jar.setCookie(cookie, url);
		return {
			url: url,
			jar: jar,
			board: board,
			pageNum: pageNum
		};
	},
	
	getPostUrl: function (board, post) {<commit_msg>Fine tune.<commit_after>var util = require('util'),
	request = require('request'),
	hp = require("htmlparser"),
	PageCrawlingStream = require('./stream/page-crawling-stream'),
	map = require('./map');

var jar = request.jar(),
	cookie = request.cookie("over18=1"),
	request = request.defaults({jar: jar});

var _text = function (node) {
	var buffer = [], str;
	_text0(buffer, node);
	str = buffer.join('');
	return str[str.length - 1] === '\n' ? str : str + '\n';
};

var _text0 = function (buffer, node) {
	if (node.type === 'text')
		buffer.push(node.data);
	else if (node.children)
		node.children.forEach(function (c) { _text0(buffer, c); });
};

var _parseComment = function (node) {
	var subtype, author, content, time;
	if (!node.children)
		return {};
	node.children.forEach(function (c) {
		var cls = c.attribs.class,
			cnt = (c.children && c.children[0] && c.children[0].data) || '';
		if (!cls)
			return;
		if (cls.indexOf('push-tag') > -1)
			subtype = 
				cnt.indexOf('推') > -1 ? 1 : 
				cnt.indexOf('噓') > -1 ? 2 : 3;
		else if (cls.indexOf('push-userid') > -1)
			author = cnt;
		else if (cls.indexOf('push-content') > -1)
			content = cnt.substring(1);
		else if (cls.indexOf('push-ipdatetime') > -1)
			time = cnt.trim();
	});
	return { type: 'comment', subtype: subtype, author: author, 
		content: content, time: time };
};

var _parseMetaline = function (node) {
	var tag, value;
	if (!node.children)
		return {};
	node.children.forEach(function (c) {
		var cls = c.attribs.class,
			cnt = (c.children && c.children[0] && c.children[0].data) || '';
		if (!cls)
			return;
		if (cls.indexOf('article-meta-tag') > -1)
			tag = cnt;
		else if (cls.indexOf('article-meta-value') > -1)
			value = cnt;
	});
	return { tag: tag, value: value };
};

var _parsePostResponse = function (data, raw) {
	var url = data.url,
		body = data.body,
		result = { board: url.board, post: url.post },
		handler, parser;
	
	handler = new hp.DefaultHandler(function (error, dom) {}, { 
		verbose: false, ignoreWhitespace: true
	});
	parser = new hp.Parser(handler);
	parser.parseComplete(body);
	
	// title
	// <meta property="og:title" content="[TITLE]" />
	var tm = /property="og:title" content="([^"]*)"/.exec(body);
	if (tm)
		result.title = tm[1];
	
	// list of content/comments
	var content = result.content = [], 
		buffer = [], 
		meta = result.meta = [];
	hp.DomUtils.getElementById('main-content', handler.dom).children.forEach(function (node) {
		var cls = node.attribs && node.attribs.class;
		
		if (cls && cls.indexOf('article-metaline') > -1) {
			meta.push(_parseMetaline(node));
			return;
		}
		
		if (cls && cls.indexOf('richcontent') > -1)
			return;
		
		if (cls && cls.indexOf('push') > -1) {
			if (buffer.length)
				content.push(buffer.join(''));
			buffer = [];
			content.push(_parseComment(node));
			return;
		}
		
		buffer.push(_text(node));
	});
	if (buffer.length)
		content.push(buffer.join(''));
	
	var tag, value;
	meta.forEach(function (pair) {
		tag = pair.tag;
		value = pair.value;
		if (tag === '時間') {
			var date = new Date(value); // TODO: might be implementation dependent
			if (date)
				result.timestamp = date.getTime();
		}
	});
	
	if (raw)
		result.raw = data.body;
	
	return result;
};

var util = {
	
	getBoardUrl: function (board, pageNum) {
		var url = "http://www.ptt.cc/bbs/" + board + 
			"/index" + (pageNum != null ? pageNum : "") + ".html"
		jar.setCookie(cookie, url);
		return {
			url: url,
			jar: jar,
			board: board,
			pageNum: pageNum
		};
	},
	
	getPostUrl: function (board, post) {
{'fraction_non_alphanumeric': 0.19450900639163277, 'fraction_numerical': 0.007844276583381755, 'mean_word_length': 3.3438485804416405, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6012552', 'n_tokens_mistral': 2760, 'n_tokens_neox': 2505, 'n_words': 580}