<commit_before>'use strict';

var _            = require('lodash'),
    db           = require('larvitdb'),
    log          = require('winston'),
    async        = require('async'),
    events       = require('events'),
    slugify      = require('larvitslugify'),
    dbmigration  = require('larvitdbmigration')({'tableName': 'blog_db_version', 'migrationScriptsPath': __dirname + '/dbmigration'}),
    eventEmitter = new events.EventEmitter(),
    dbChecked    = false;

// Handle database migrations
dbmigration(function(err) {
	if (err) {
		log.error('larvitblog: createTablesIfNotExists() - Database error: ' + err.message);
		return;
	}

	dbChecked = true;
	eventEmitter.emit('checked');
});

/**
 * Get blog entries
 *
 * @param obj options - { // All options are optional!
 *                        'langs': ['sv', 'en'],
 *                        'slugs': ['blu', 'bla'],
 *                        'publishedAfter': dateObj,
 *                        'publishedBefore': dateObj,
 *                        'tags': ['dks', 'ccc'],
 *                        'ids': [32,4],
 *                        'limit': 10,
 *                        'offset': 20
 *                      }
 * @param func cb - callback(err, entries)
 */
function getEntries(options, cb) {
	var tmpEntries = {},
	    dbFields   = [],
	    entries    = [],
	    sql,
	    i;

	if (typeof options === 'function') {
		cb      = options;
		options = {};
	}

	log.debug('larvitblog: getEntries() - Called with options: "' + JSON.stringify(options) + '"');

	// Make sure options that should be arrays actually are arrays
	// This will simplify our lives in the SQL builder below
	if (options.langs !== undefined && ! (options.langs instanceof Array))
		options.langs = [options.langs];

	if (options.ids !== undefined && ! (options.ids instanceof Array))
		options.ids = [options.ids];

	if (options.slugs !== undefined && ! (options.slugs instanceof Array))
		options.slugs = [options.slugs];

	if (options.tags !== undefined && ! (options.tags instanceof Array))
		options.tags = [options.tags];

	// Make sure there is an invalid ID in the id list if it is empty
	// Since the most logical thing to do is replying with an empty set
	if (options.ids instanceof Array && options.ids.length === 0)
		options.ids.push(- 1);

	if (options.limit === undefined)
		options.limit = 10;

	// Make sure the database tables exists before going further!
	if ( ! dbChecked) {
		log.debug('larvitblog: getEntries() - Database not checked, rerunning this method when event have been emitted.');
		eventEmitter.on('checked', function() {
			log.debug('larvitblog: getEntries() - Database check event received, rerunning getEntries().');
			getEntries(options, cb);
		});

		return;
	}

	sql  = 'SELECT ed.*, e.id, e.created, e.published, GROUP_CONCAT(t.content) AS tags, GROUP_CONCAT(i.uri) AS images\n';
	sql += 'FROM blog_entries e\n';
	sql += '	LEFT JOIN blog_entriesData       ed ON ed.entryId = e.id\n';
	sql += '	LEFT JOIN blog_entriesDataTags   t  ON t.entryId  = e.id AND t.lang = ed.lang\n';
	sql += '	LEFT JOIN blog_entriesDataImages i  ON i.entryId  = e.id\n';
	sql += 'WHERE 1 + 1\n';

	// Only get post contents with selected languages
	if (options.langs !== undefined) {<commit_msg>Fixed duplicate issue in SQL<commit_after>'use strict';

var _            = require('lodash'),
    db           = require('larvitdb'),
    log          = require('winston'),
    async        = require('async'),
    events       = require('events'),
    slugify      = require('larvitslugify'),
    dbmigration  = require('larvitdbmigration')({'tableName': 'blog_db_version', 'migrationScriptsPath': __dirname + '/dbmigration'}),
    eventEmitter = new events.EventEmitter(),
    dbChecked    = false;

// Handle database migrations
dbmigration(function(err) {
	if (err) {
		log.error('larvitblog: createTablesIfNotExists() - Database error: ' + err.message);
		return;
	}

	dbChecked = true;
	eventEmitter.emit('checked');
});

/**
 * Get blog entries
 *
 * @param obj options - { // All options are optional!
 *                        'langs': ['sv', 'en'],
 *                        'slugs': ['blu', 'bla'],
 *                        'publishedAfter': dateObj,
 *                        'publishedBefore': dateObj,
 *                        'tags': ['dks', 'ccc'],
 *                        'ids': [32,4],
 *                        'limit': 10,
 *                        'offset': 20
 *                      }
 * @param func cb - callback(err, entries)
 */
function getEntries(options, cb) {
	var tmpEntries = {},
	    dbFields   = [],
	    entries    = [],
	    sql,
	    i;

	if (typeof options === 'function') {
		cb      = options;
		options = {};
	}

	log.debug('larvitblog: getEntries() - Called with options: "' + JSON.stringify(options) + '"');

	// Make sure options that should be arrays actually are arrays
	// This will simplify our lives in the SQL builder below
	if (options.langs !== undefined && ! (options.langs instanceof Array))
		options.langs = [options.langs];

	if (options.ids !== undefined && ! (options.ids instanceof Array))
		options.ids = [options.ids];

	if (options.slugs !== undefined && ! (options.slugs instanceof Array))
		options.slugs = [options.slugs];

	if (options.tags !== undefined && ! (options.tags instanceof Array))
		options.tags = [options.tags];

	// Make sure there is an invalid ID in the id list if it is empty
	// Since the most logical thing to do is replying with an empty set
	if (options.ids instanceof Array && options.ids.length === 0)
		options.ids.push(- 1);

	if (options.limit === undefined)
		options.limit = 10;

	// Make sure the database tables exists before going further!
	if ( ! dbChecked) {
		log.debug('larvitblog: getEntries() - Database not checked, rerunning this method when event have been emitted.');
		eventEmitter.on('checked', function() {
			log.debug('larvitblog: getEntries() - Database check event received, rerunning getEntries().');
			getEntries(options, cb);
		});

		return;
	}

	sql  = 'SELECT ed.*, e.id, e.created, e.published, GROUP_CONCAT(DISTINCT t.content) AS tags, GROUP_CONCAT(DISTINCT i.uri) AS images\n';
	sql += 'FROM blog_entries e\n';
	sql += '	LEFT JOIN blog_entriesData       ed ON ed.entryId = e.id\n';
	sql += '	LEFT JOIN blog_entriesDataTags   t  ON t.entryId  = e.id AND t.lang = ed.lang\n';
	sql += '	LEFT JOIN blog_entriesDataImages i  ON i.entryId  = e.id\n';
	sql += 'WHERE 1 + 1\n';

	// Only get post contents with selected languages
	if (options.langs !== undefined) {
{'fraction_non_alphanumeric': 0.14462620394435102, 'fraction_numerical': 0.003974927381134383, 'mean_word_length': 2.6364647026125625, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6547908', 'n_tokens_mistral': 2089, 'n_tokens_neox': 1936, 'n_words': 572}