<commit_before>import axios from 'axios';
import mongoose from 'mongoose';
import joi from 'joi';

import db from '../utils/db';

import RSS from '../models/rss';
import Article from '../models/article';
import logger from '../utils/logger';
import { ProcessSocialQueue } from '../asyncTasks';
import { timeIt } from '../utils/statsd';
import { fetchSocialScore } from '../utils/social';
import { setupAxiosRedirectInterceptor } from '../utils/axios';
import { ensureEncoded } from '../utils/urls';

if (require.main === module) {
	setupAxiosRedirectInterceptor(axios);
}

// connect the handler to the queue
logger.info('Starting the Social worker');

ProcessSocialQueue(25, socialProcessor);

const streamQueueSettings = { removeOnComplete: true, removeOnFail: true };

const joiObjectId = joi.alternatives().try(
	joi.string().length(12),
	joi.string().length(24).regex(/^[0-9a-fA-F]{24}$/)
);
const joiUrl = joi.string().uri({ scheme: ['http', 'https'], allowQuerySquareBrackets: true });

const schema = joi.object().keys({
	rss: joiObjectId.required(),
	articles: joi.array().min(1).required(),
});
const itemSchema = joi.object().keys({
	id: joiObjectId.required(),
	link: joiUrl,
	commentUrl: joi.any(),
});
<commit_msg>Increase social task concurrency<commit_after>import axios from 'axios';
import mongoose from 'mongoose';
import joi from 'joi';

import db from '../utils/db';

import RSS from '../models/rss';
import Article from '../models/article';
import logger from '../utils/logger';
import { ProcessSocialQueue } from '../asyncTasks';
import { timeIt } from '../utils/statsd';
import { fetchSocialScore } from '../utils/social';
import { setupAxiosRedirectInterceptor } from '../utils/axios';
import { ensureEncoded } from '../utils/urls';

if (require.main === module) {
	setupAxiosRedirectInterceptor(axios);
}

// connect the handler to the queue
logger.info('Starting the Social worker');

ProcessSocialQueue(100, socialProcessor);

const streamQueueSettings = { removeOnComplete: true, removeOnFail: true };

const joiObjectId = joi.alternatives().try(
	joi.string().length(12),
	joi.string().length(24).regex(/^[0-9a-fA-F]{24}$/)
);
const joiUrl = joi.string().uri({ scheme: ['http', 'https'], allowQuerySquareBrackets: true });

const schema = joi.object().keys({
	rss: joiObjectId.required(),
	articles: joi.array().min(1).required(),
});
const itemSchema = joi.object().keys({
	id: joiObjectId.required(),
	link: joiUrl,
	commentUrl: joi.any(),
});

{'fraction_non_alphanumeric': 0.1752827140549273, 'fraction_numerical': 0.009289176090468497, 'mean_word_length': 5.041463414634146, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3187222', 'n_tokens_mistral': 806, 'n_tokens_neox': 785, 'n_words': 201}