<commit_before>var fs = require('fs'),
	_ = require('lodash'),
	path = require('path'),
	xmldom = require('xmldom');

var IGNORE_TAGS = [
	'buildmobile.com',
	'portfolio',
	'blog',
	'uncategorized',
	'mobie'
];

var serializer = new xmldom.XMLSerializer(),
	doc;

module.exports = function main(exportFile, octopress) {

	// validate input
	if (!exportFile) {
		throw new Error('exportFile must be defined');
	} else if (!fs.existsSync(exportFile)) {
		throw new Error('exportFile does not exist');
	}

	octopress = path.resolve(octopress || '.');
	if (!fs.existsSync(octopress)) {
		throw new Error('octopress does not exist');
	}

	postsDir = path.join(octopress, 'source', '_posts');
	if (!fs.existsSync(postsDir)) {
		throw new Error('octopress source/_posts dir not found');
	}

	// read export xml
	var xml = fs.readFileSync(exportFile, 'utf8'),
		source = fs.readFileSync(path.join(__dirname, 'src', 'post.markdown'));
	doc = new xmldom.DOMParser().parseFromString(xml);

	// create tag map
	var tagMap = {};
	getElements({ name: 'table', attr: 'name', attrValue: 'wp_terms' }).filter(function(table) {
		return getElement({
			node: table,
			name: 'column',
			attr: 'name',
			attrValue: 'term_group',
			value: '0'
		});
	}).forEach(function(term) {
		var col = function(n) { return getColumnValue(term, n); };
		tagMap[col('term_id')] = col('name');
	});

	// create taxonomy map
	var taxonomyMap = {};
	getElements({ name: 'table', attr: 'name', attrValue: 'wp_term_taxonomy' }).filter(function(table) {
		return getElement({
			node: table,
			name: 'column',
			attr: 'name'
		});
	}).forEach(function(term) {
		var col = function(n) { return getColumnValue(term, n); };
		taxonomyMap[col('term_taxonomy_id')] = col('term_id');
	});

	// find posts
	getElements('table').filter(function(table) {
		var has = function(o) {
			return !!getElement(_.extend({
				node: table,
				name: 'column',
				attr: 'name'
			}, o));
		};
		return has({ attrValue: 'post_type', value: 'post' }) &&
			has({ attrValue: 'post_status', value: 'publish' });
	}).forEach(function(elem) {
		var col = function(n) { return getColumnValue(elem, n); };
		var post = {
			categories: ['archive'],
			content: col('post_content')
				.replace(/&amp;/g, "&")
				.replace(/&lt;/g, "<")
				.replace(/&gt;/g, ">")
				.replace(/&quot;/g, "\"")
				.replace(/&#039;/g, "'"),
			date: col('post_date'),
			id: col('ID'),
			status: col('post_status'),
			title: col('post_title')
		};

		// find tags for post
		var tags = [];
		getElements({
			name: 'table',
			attr: 'name',
			attrValue: 'wp_term_relationships'
		}).forEach(function(elem) {
			var col = function(n) { return getColumnValue(elem, n); };
			if (col('object_id') === post.id) {
				var tag = tagMap[taxonomyMap[col('term_taxonomy_id')]].toLowerCase();
				if (!_.contains(IGNORE_TAGS, tag)) {
					post.categories.push(tag);<commit_msg>shorten date<commit_after>var fs = require('fs'),
	_ = require('lodash'),
	path = require('path'),
	xmldom = require('xmldom');

var IGNORE_TAGS = [
	'buildmobile.com',
	'portfolio',
	'blog',
	'uncategorized',
	'mobie'
];

var serializer = new xmldom.XMLSerializer(),
	doc;

module.exports = function main(exportFile, octopress) {

	// validate input
	if (!exportFile) {
		throw new Error('exportFile must be defined');
	} else if (!fs.existsSync(exportFile)) {
		throw new Error('exportFile does not exist');
	}

	octopress = path.resolve(octopress || '.');
	if (!fs.existsSync(octopress)) {
		throw new Error('octopress does not exist');
	}

	postsDir = path.join(octopress, 'source', '_posts');
	if (!fs.existsSync(postsDir)) {
		throw new Error('octopress source/_posts dir not found');
	}

	// read export xml
	var xml = fs.readFileSync(exportFile, 'utf8'),
		source = fs.readFileSync(path.join(__dirname, 'src', 'post.markdown'));
	doc = new xmldom.DOMParser().parseFromString(xml);

	// create tag map
	var tagMap = {};
	getElements({ name: 'table', attr: 'name', attrValue: 'wp_terms' }).filter(function(table) {
		return getElement({
			node: table,
			name: 'column',
			attr: 'name',
			attrValue: 'term_group',
			value: '0'
		});
	}).forEach(function(term) {
		var col = function(n) { return getColumnValue(term, n); };
		tagMap[col('term_id')] = col('name');
	});

	// create taxonomy map
	var taxonomyMap = {};
	getElements({ name: 'table', attr: 'name', attrValue: 'wp_term_taxonomy' }).filter(function(table) {
		return getElement({
			node: table,
			name: 'column',
			attr: 'name'
		});
	}).forEach(function(term) {
		var col = function(n) { return getColumnValue(term, n); };
		taxonomyMap[col('term_taxonomy_id')] = col('term_id');
	});

	// find posts
	getElements('table').filter(function(table) {
		var has = function(o) {
			return !!getElement(_.extend({
				node: table,
				name: 'column',
				attr: 'name'
			}, o));
		};
		return has({ attrValue: 'post_type', value: 'post' }) &&
			has({ attrValue: 'post_status', value: 'publish' });
	}).forEach(function(elem) {
		var col = function(n) { return getColumnValue(elem, n); };
		var post = {
			categories: ['archive'],
			content: col('post_content')
				.replace(/&amp;/g, "&")
				.replace(/&lt;/g, "<")
				.replace(/&gt;/g, ">")
				.replace(/&quot;/g, "\"")
				.replace(/&#039;/g, "'"),
			date: col('post_date').replace(/.{3}$/,''),
			id: col('ID'),
			status: col('post_status'),
			title: col('post_title')
		};

		// find tags for post
		var tags = [];
		getElements({
			name: 'table',
			attr: 'name',
			attrValue: 'wp_term_relationships'
		}).forEach(function(elem) {
			var col = function(n) { return getColumnValue(elem, n); };
			if (col('object_id') === post.id) {
				var tag = tagMap[taxonomyMap[col('term_taxonomy_id')]].toLowerCase();
				if (!_.contains(IGNORE_TAGS, tag)) {
					post.categories.push(tag);
{'fraction_non_alphanumeric': 0.19645011201102877, 'fraction_numerical': 0.001895571256246769, 'mean_word_length': 4.091228070175439, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '28299', 'n_tokens_mistral': 2294, 'n_tokens_neox': 2061, 'n_words': 452}