<commit_before>'use strict';

/* globals module, require */

const db = require.main.require('./src/database');
const winston = require.main.require('winston');
const engine = require('solr-client');
const async = require('async');

const LRU = require('lru-cache');
const titleCache = LRU({ max: 20, maxAge: 1000 * 60 * 20 });	// Remember the last 20 searches in the past twenty minutes
const postCache = LRU({ max: 20, maxAge: 1000 * 60 * 20 });	// Remember the last 20 searches in the past twenty minutes

const topics = require.main.require('./src/topics');
const posts = require.main.require('./src/posts');
const batch = require.main.require('./src/batch');
const utils = require('./lib/utils');

const Solr = {
	/*
		Defaults configs:
		host: localhost
		port: 8983
		core: ''
		path: '/solr'
		enabled: undefined (false)
		titleField: 'title_t'
		contentField: 'description_t',
		secure: false
	*/
	config: {},	// default is localhost:8983, '' core, '/solr' path
	client: undefined,
	indexStatus: {
		running: false,
		current: 0,
		total: 0,
		message: 'Initializing',
	},
};

Solr.init = function (data, callback) {
	const pluginMiddleware = require('./middleware');
	const render = function (req, res) {
		// Regenerate csrf token
		const token = req.csrfToken();

		res.render('admin/plugins/solr', {
			ping: res.locals.ping,
			enabled: res.locals.enabled,
			stats: res.locals.stats,
			csrf: token,
			running: Solr.indexStatus.running,
		});
	};

	data.router.get('/admin/plugins/solr', data.middleware.applyCSRF, data.middleware.admin.buildHeader, pluginMiddleware.ping, pluginMiddleware.getEnabled, pluginMiddleware.getStats, render);
	data.router.get('/api/admin/plugins/solr', data.middleware.applyCSRF, pluginMiddleware.ping, pluginMiddleware.getEnabled, pluginMiddleware.getStats, render);

	// Utility
	data.router.post('/admin/plugins/solr/rebuild', Solr.rebuildIndex);
	data.router.get('/admin/plugins/solr/rebuildProgress', Solr.getIndexProgress);
	data.router.post('/admin/plugins/solr/toggle', Solr.toggle);
	data.router.delete('/admin/plugins/solr/flush', Solr.flush);
	data.router.delete('/admin/plugins/solr/cache', Solr.dropCaches);

	Solr.getSettings(Solr.connect);

	callback();
};

Solr.ping = function (callback) {
	if (Solr.client) {
		Solr.client.ping(callback);
	} else {
		callback(new Error('not-connected'));
	}
};

Solr.checkConflict = function () {
	return !!module.parent.exports.libraries['nodebb-plugin-dbsearch'];
};

Solr.getNotices = function (notices, callback) {
	Solr.ping(function (err) {
		const solrNotices = [
			{ done: !err, doneText: 'Solr connection OK', notDoneText: 'Could not connect to Solr server' },
			{ done: parseInt(Solr.config.enabled, 10) || false, doneText: 'Solr Indexing Enabled', notDoneText: 'Solr Indexing Disabled' },
		];

		callback(null, notices.concat(solrNotices));
	});
};

Solr.getSettings = function (callback) {
	db.getObject('settings:solr', function (err, config) {
		Solr.config = {};
		if (!err) {
			for (const k in config) {
				if (config.hasOwnProperty(k) && config[k].length && !Solr.config.hasOwnProperty(k)) {
					Solr.config[k] = config[k];
				}
			}
		} else {
			winston.error('[plugin:solr] Could not fetch settings, assuming defaults.');
		}

		callback();
	});
};

Solr.getRecordCount = function (callback) {
	const query = Solr.client.createQuery().q('*:*').start(0).rows(0);

	Solr.client.search(query, function (err, obj) {
		if (!err && obj && obj.response) {
			callback(undefined, obj.response.numFound);
		} else {
			callback(err, 0);
		}
	});
};

Solr.getTopicCount = function (callback) {
	const query = Solr.client.createQuery().q((Solr.config.titleField || 'title_t') + ':*').start(0).rows(0);

	Solr.client.search(query, function (err, obj) {
		if (!err && obj && obj.response) {
			callback(undefined, obj.response.numFound);
		} else {
			callback(err, 0);
		}
	});
};

Solr.connect = function () {
	if (Solr.client) {
		delete Solr.client;
	}

	Solr.config = {
		...Solr.config,
		secure: Solr.config.secure && Solr.config.secure === 'on',
	};

	Solr.client = engine.createClient(Solr.config);

	if (Solr.config.username && Solr.config.password) {
		Solr.client.basicAuth(Solr.config.username, Solr.config.password);
	}
};

Solr.adminMenu = function (custom_header, callback) {
	custom_header.plugins.push({
		route: '/plugins/solr',
		icon: 'fa-search',
		name: 'Apache Solr',
	});

	callback(null, custom_header);
};

Solr.search = function (data, callback) {
	if (Solr.checkConflict()) {
		// The dbsearch plugin was detected, abort search!
		winston.warn('[plugin/solr] Another search plugin (dbsearch) is enabled, so search via Solr was aborted.');
		return callback(null, data);
	}
	const isTopic = data.index === 'topic';
	const field = isTopic ? 'tid_i' : 'pid_i';
	const term = utils.addFiltersToTerm(data.content, data);

	// Determine which cache to use
	const cache = isTopic ? titleCache : postCache;

	if (cache.has(term)) {
		callback(null, cache.get(term));
	} else {
		const fields = {};

		// Populate Fields
		if (isTopic) { fields[Solr.config.titleField || 'title_t'] = 1; } else { fields[Solr.config.contentField || 'description_t'] = 1; }

		const query = Solr.client.createQuery()
			.q(term)
			.edismax()
			.qf(fields)
			.start(0)
			.rows(500);

		Solr.ping(function (err) {
			if (err) {
				// Stop use search engine when connection failed
				winston.warn('[plugins/solr] Could not connect to Solr server');
				callback(null, data);
			} else {
				Solr.client.search(query, function (err, obj) {
					if (err) {
						return callback(err);
					} else if (obj && obj.response && obj.response.docs.length > 0) {
						const payload = obj.response.docs.map(function (result) {
							return result[field];
						}).filter(Boolean);

						callback(null, payload);
						cache.set(term, payload);
					} else {
						callback(null, []);
						cache.set(term, []);
					}

					winston.verbose('[plugin/solr] Search (' + data.index + ') for "' + data.content + '" returned ' + obj.response.docs.length + ' results');
				});
			}
		});
	}
};

Solr.searchTopic = function (data, callback) {
	const tid = data.tid;<commit_msg>Added workaround for mysterious unescaped = character (that turns into ^ in search query) (#57)<commit_after>'use strict';

/* globals module, require */

const db = require.main.require('./src/database');
const winston = require.main.require('winston');
const engine = require('solr-client');
const async = require('async');

const LRU = require('lru-cache');
const titleCache = LRU({ max: 20, maxAge: 1000 * 60 * 20 });	// Remember the last 20 searches in the past twenty minutes
const postCache = LRU({ max: 20, maxAge: 1000 * 60 * 20 });	// Remember the last 20 searches in the past twenty minutes

const topics = require.main.require('./src/topics');
const posts = require.main.require('./src/posts');
const batch = require.main.require('./src/batch');
const utils = require('./lib/utils');

const Solr = {
	/*
		Defaults configs:
		host: localhost
		port: 8983
		core: ''
		path: '/solr'
		enabled: undefined (false)
		titleField: 'title_t'
		contentField: 'description_t',
		secure: false
	*/
	config: {},	// default is localhost:8983, '' core, '/solr' path
	client: undefined,
	indexStatus: {
		running: false,
		current: 0,
		total: 0,
		message: 'Initializing',
	},
};

Solr.init = function (data, callback) {
	const pluginMiddleware = require('./middleware');
	const render = function (req, res) {
		// Regenerate csrf token
		const token = req.csrfToken();

		res.render('admin/plugins/solr', {
			ping: res.locals.ping,
			enabled: res.locals.enabled,
			stats: res.locals.stats,
			csrf: token,
			running: Solr.indexStatus.running,
		});
	};

	data.router.get('/admin/plugins/solr', data.middleware.applyCSRF, data.middleware.admin.buildHeader, pluginMiddleware.ping, pluginMiddleware.getEnabled, pluginMiddleware.getStats, render);
	data.router.get('/api/admin/plugins/solr', data.middleware.applyCSRF, pluginMiddleware.ping, pluginMiddleware.getEnabled, pluginMiddleware.getStats, render);

	// Utility
	data.router.post('/admin/plugins/solr/rebuild', Solr.rebuildIndex);
	data.router.get('/admin/plugins/solr/rebuildProgress', Solr.getIndexProgress);
	data.router.post('/admin/plugins/solr/toggle', Solr.toggle);
	data.router.delete('/admin/plugins/solr/flush', Solr.flush);
	data.router.delete('/admin/plugins/solr/cache', Solr.dropCaches);

	Solr.getSettings(Solr.connect);

	callback();
};

Solr.ping = function (callback) {
	if (Solr.client) {
		Solr.client.ping(callback);
	} else {
		callback(new Error('not-connected'));
	}
};

Solr.checkConflict = function () {
	return !!module.parent.exports.libraries['nodebb-plugin-dbsearch'];
};

Solr.getNotices = function (notices, callback) {
	Solr.ping(function (err) {
		const solrNotices = [
			{ done: !err, doneText: 'Solr connection OK', notDoneText: 'Could not connect to Solr server' },
			{ done: parseInt(Solr.config.enabled, 10) || false, doneText: 'Solr Indexing Enabled', notDoneText: 'Solr Indexing Disabled' },
		];

		callback(null, notices.concat(solrNotices));
	});
};

Solr.getSettings = function (callback) {
	db.getObject('settings:solr', function (err, config) {
		Solr.config = {};
		if (!err) {
			for (const k in config) {
				if (config.hasOwnProperty(k) && config[k].length && !Solr.config.hasOwnProperty(k)) {
					Solr.config[k] = config[k];
				}
			}
		} else {
			winston.error('[plugin:solr] Could not fetch settings, assuming defaults.');
		}

		callback();
	});
};

Solr.getRecordCount = function (callback) {
	const query = Solr.client.createQuery().q('*:*').start(0).rows(0);

	Solr.client.search(query, function (err, obj) {
		if (!err && obj && obj.response) {
			callback(undefined, obj.response.numFound);
		} else {
			callback(err, 0);
		}
	});
};

Solr.getTopicCount = function (callback) {
	const query = Solr.client.createQuery().q((Solr.config.titleField || 'title_t') + ':*').start(0).rows(0);

	Solr.client.search(query, function (err, obj) {
		if (!err && obj && obj.response) {
			callback(undefined, obj.response.numFound);
		} else {
			callback(err, 0);
		}
	});
};

Solr.connect = function () {
	if (Solr.client) {
		delete Solr.client;
	}

	Solr.config = {
		...Solr.config,
		secure: Solr.config.secure && Solr.config.secure === 'on',
	};

	Solr.client = engine.createClient(Solr.config);

	if (Solr.config.username && Solr.config.password) {
		Solr.client.basicAuth(Solr.config.username, Solr.config.password);
	}
};

Solr.adminMenu = function (custom_header, callback) {
	custom_header.plugins.push({
		route: '/plugins/solr',
		icon: 'fa-search',
		name: 'Apache Solr',
	});

	callback(null, custom_header);
};

Solr.search = function (data, callback) {
	if (Solr.checkConflict()) {
		// The dbsearch plugin was detected, abort search!
		winston.warn('[plugin/solr] Another search plugin (dbsearch) is enabled, so search via Solr was aborted.');
		return callback(null, data);
	}
	const isTopic = data.index === 'topic';
	const field = isTopic ? 'tid_i' : 'pid_i';
	const term = utils.addFiltersToTerm(data.content, data);

	// Determine which cache to use
	const cache = isTopic ? titleCache : postCache;

	if (cache.has(term)) {
		callback(null, cache.get(term));
	} else {
		const fields = {};

		// Populate Fields
		if (isTopic) { fields[Solr.config.titleField || 'title_t'] = 1; } else { fields[Solr.config.contentField || 'description_t'] = 1; }

		const query = Solr.client.createQuery()
			.q(term)
			.edismax()
			.qf(fields)
			.start(0)
			.rows(500);

		Solr.ping(function (err) {
			if (err) {
				// Stop use search engine when connection failed
				winston.warn('[plugins/solr] Could not connect to Solr server');
				callback(null, data);
			} else {

				// Working around: https://github.com/lbdremy/solr-node-client/issues/197
				query.parameters = query.parameters.map(param => param.replace('^1', '%5E1'));

				Solr.client.search(query, function (err, obj) {
					if (err) {
						return callback(err);
					} else if (obj && obj.response && obj.response.docs.length > 0) {
						const payload = obj.response.docs.map(function (result) {
							return result[field];
						}).filter(Boolean);

						callback(null, payload);
						cache.set(term, payload);
					} else {
						callback(null, []);
						cache.set(term, []);
					}

					winston.verbose('[plugin/solr] Search (' + data.index + ') for "' + data.content + '" returned ' + obj.response.docs.length + ' results');
				});
			}
		});
	}
};

Solr.searchTopic = function (data, callback) {
	const tid = data.tid;
{'fraction_non_alphanumeric': 0.1676851043567971, 'fraction_numerical': 0.008412030791207047, 'mean_word_length': 3.861882716049383, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3977597', 'n_tokens_mistral': 4840, 'n_tokens_neox': 4257, 'n_words': 930}