<commit_before>/**
 * Code for using Google's Perspective API for filters.
 * @author mia-pi-git
 */
import {ProcessManager, Net, Repl} from '../../lib';
import {Config} from '../config-loader';
import {toID} from '../../sim/dex-data';

// 20m. this is mostly here so we can use Monitor.slow()
const PM_TIMEOUT = 20 * 60 * 1000;
export const ATTRIBUTES = {
	"SEVERE_TOXICITY": {},
	"TOXICITY": {},
	"IDENTITY_ATTACK": {},
	"INSULT": {},
	"PROFANITY": {},
	"THREAT": {},
	"SEXUALLY_EXPLICIT": {},
	"FLIRTATION": {},
};


export interface PerspectiveRequest {
	languages: string[];
	requestedAttributes: AnyObject;
	comment: {text: string};
}

function time() {
	return Math.floor(Date.now() / 1000);
}

export class Limiter {
	readonly max: number;
	readonly period: number;
	lastTick = time();
	counts: number[] = [];
	constructor(max: number, period: number) {
		this.max = max;
		this.period = period;
	}
	shouldRequest() {
		const now = time();
		if (this.lastTick === now) {
			// don't increment if it's already too much
			if (this.sum() > this.max) return false;
		}
		this.increment(time());
		return this.sum() < this.max;
	}
	sum() {
		return (this.counts.reduce((a, b) => a + b) / this.counts.length);
	}
	increment(now: number) {
		const last = this.lastTick;
		this.lastTick = now;
		const diff = now - last;
		for (let i = 0; i < diff; i++) this.counts.push(0);
		if (this.counts.length > this.period) {
			for (let i = 0; i < (this.counts.length - this.period); i++) {
				this.counts.shift();
			}
		}
		this.counts[this.counts.length - 1]++;
	}
}

function isCommon(message: string) {
	message = message.toLowerCase().replace(/\?!\., ;:/g, '');
	return ['gg', 'wp', 'ggwp', 'gl', 'hf', 'glhf', 'hello'].includes(message);
}

let throttleTime: number | null = null;
export const limiter = new Limiter(10, 5);
export const PM = new ProcessManager.QueryProcessManager<string, Record<string, number> | null>(module, async text => {
	if (isCommon(text) || !limiter.shouldRequest()) return null;
	if (throttleTime && (Date.now() - throttleTime < 10000)) {
		return null;
	}
	if (throttleTime) throttleTime = null;

	const requestData: PerspectiveRequest = {
		// todo - support 'es', 'it', 'pt', 'fr' - use user.language? room.settings.language...?
		languages: ['en'],
		requestedAttributes: ATTRIBUTES,<commit_msg>Artemis: Remove average limiter<commit_after>/**
 * Code for using Google's Perspective API for filters.
 * @author mia-pi-git
 */
import {ProcessManager, Net, Repl} from '../../lib';
import {Config} from '../config-loader';
import {toID} from '../../sim/dex-data';

// 20m. this is mostly here so we can use Monitor.slow()
const PM_TIMEOUT = 20 * 60 * 1000;
export const ATTRIBUTES = {
	"SEVERE_TOXICITY": {},
	"TOXICITY": {},
	"IDENTITY_ATTACK": {},
	"INSULT": {},
	"PROFANITY": {},
	"THREAT": {},
	"SEXUALLY_EXPLICIT": {},
	"FLIRTATION": {},
};


export interface PerspectiveRequest {
	languages: string[];
	requestedAttributes: AnyObject;
	comment: {text: string};
}

function time() {
	return Math.floor(Date.now() / 1000);
}

export class Limiter {
	readonly max: number;
	lastTick = time();
	count = 0;
	constructor(max: number) {
		this.max = max;
	}
	shouldRequest() {
		const now = time();
		if (this.lastTick !== now) {
			this.count = 0;
			this.lastTick = now;
		}
		this.count++;
		return this.count < this.max;
	}
}

function isCommon(message: string) {
	message = message.toLowerCase().replace(/\?!\., ;:/g, '');
	return ['gg', 'wp', 'ggwp', 'gl', 'hf', 'glhf', 'hello'].includes(message);
}

let throttleTime: number | null = null;
export const limiter = new Limiter(15);
export const PM = new ProcessManager.QueryProcessManager<string, Record<string, number> | null>(module, async text => {
	if (isCommon(text) || !limiter.shouldRequest()) return null;
	if (throttleTime && (Date.now() - throttleTime < 10000)) {
		return null;
	}
	if (throttleTime) throttleTime = null;

	const requestData: PerspectiveRequest = {
		// todo - support 'es', 'it', 'pt', 'fr' - use user.language? room.settings.language...?
		languages: ['en'],
		requestedAttributes: ATTRIBUTES,
{'fraction_non_alphanumeric': 0.18339016074037992, 'fraction_numerical': 0.011933755479785679, 'mean_word_length': 3.9422382671480145, 'pattern_counts': {'":': 16, '<': 13, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1040702', 'n_tokens_mistral': 1553, 'n_tokens_neox': 1442, 'n_words': 380}