<commit_before>/* eslint-disable camelcase */
const config = require('./../config');
const elasticsearch = require('elasticsearch');

const client = new elasticsearch.Client({ host: `${config.elasticsearch.host}:${config.elasticsearch.port}` });
const knex = require('knex')(config.db);

const getRecords = () =>
	knex('torrents')
		.where({ searchUpdate: false })
		.limit(config.search.limit);

const parseRecord = (record) => {
	const parsedRecord = {
		...record,
		files: [],
		search: record.name
			? record.name
					.replace(/_/gu, ' ')
					.replace(/ /gu, ' ')
					.replace(/\./gu, ' ')
					.replace(/-/gu, ' ')
			: '',
	};

	try {
		parsedRecord.files = record.files ? JSON.parse(record.files) : [];
		parsedRecord.files = parsedRecord.files.map((file) => ({
			length: file.length,
			path: file.path.join('/'),
		}));
	} catch (error) {
		// Do nothing - files is too long
	}

	return parsedRecord;
};

const update = async (records) => {
	const parsedRecords = records.filter(({ name }) => Boolean(name)).map(parseRecord);
	const body = parsedRecords.reduce(
		(result, record) => [
			...result,
			{
				update: {
					_id: record.infohash,
					_index: 'torrents',
					_retry_on_conflict: 3,
					_type: 'hash',
				},
			},
			{
				doc: record,
				doc_as_upsert: true,
			},
		],
		[],
	);

	const response = await client.bulk({ body });

	if (response.errors) {
		const errors = response.items.filter((item) => item.update.error);

		console.log(errors);
	}

	await knex('torrents')
		.update({
			searchUpdate: true,<commit_msg>disable `_retry_on_conflict` for compatibility<commit_after>/* eslint-disable camelcase */
const config = require('./../config');
const elasticsearch = require('elasticsearch');

const client = new elasticsearch.Client({ host: `${config.elasticsearch.host}:${config.elasticsearch.port}` });
const knex = require('knex')(config.db);

const getRecords = () =>
	knex('torrents')
		.where({ searchUpdate: false })
		.limit(config.search.limit);

const parseRecord = (record) => {
	const parsedRecord = {
		...record,
		files: [],
		search: record.name
			? record.name
					.replace(/_/gu, ' ')
					.replace(/ /gu, ' ')
					.replace(/\./gu, ' ')
					.replace(/-/gu, ' ')
			: '',
	};

	try {
		parsedRecord.files = record.files ? JSON.parse(record.files) : [];
		parsedRecord.files = parsedRecord.files.map((file) => ({
			length: file.length,
			path: file.path.join('/'),
		}));
	} catch (error) {
		// Do nothing - files is too long
	}

	return parsedRecord;
};

const update = async (records) => {
	const parsedRecords = records.filter(({ name }) => Boolean(name)).map(parseRecord);
	const body = parsedRecords.reduce(
		(result, record) => [
			...result,
			{
				update: {
					_id: record.infohash,
					_index: 'torrents',
					//_retry_on_conflict: 3,
					_type: 'hash',
				},
			},
			{
				doc: record,
				doc_as_upsert: true,
			},
		],
		[],
	);

	const response = await client.bulk({ body });

	if (response.errors) {
		const errors = response.items.filter((item) => item.update.error);

		console.log(errors);
	}

	await knex('torrents')
		.update({
			searchUpdate: true,
{'fraction_non_alphanumeric': 0.18823529411764706, 'fraction_numerical': 0.0006359300476947536, 'mean_word_length': 3.1286089238845145, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5235363', 'n_tokens_mistral': 1227, 'n_tokens_neox': 1073, 'n_words': 212}