<commit_before>'use strict';

const fetch = require('node-fetch');
const { addonDefinitionSchema } = require('./addon-schema');

class Addon {
    constructor(definition, { getLogger }) {
        this.logger = getLogger(`addon/${definition.name}`);
        const { error } = addonDefinitionSchema.validate(definition);
        if (error) {
            this.logger.warn(
                `Could not load addon provider ${definition.name}`,
                error,
            );
            throw error;
        }
        this._name = definition.name;
        this._definition = definition;
    }

    get name() {
        return this._name;
    }

    get definition() {
        return this._definition;
    }

    async fetchRetry(url, options = {}, retries = 1, backoff = 300) {
        const retryCodes = [408, 500, 502, 503, 504, 522, 524];
        const res = await fetch(url, options);
        if (res.ok) {
            return res;
        }
        if (retries > 0 && retryCodes.includes(res.status)) {
            setTimeout(() => {
                return this.fetchRetry(url, options, retries - 1, backoff * 2);
            }, backoff);
        }
        return res;
    }
}

module.exports = Addon;<commit_msg>fix: not crash if addon http post throws (#738)<commit_after>'use strict';

const fetch = require('node-fetch');
const { addonDefinitionSchema } = require('./addon-schema');

class Addon {
    constructor(definition, { getLogger }) {
        this.logger = getLogger(`addon/${definition.name}`);
        const { error } = addonDefinitionSchema.validate(definition);
        if (error) {
            this.logger.warn(
                `Could not load addon provider ${definition.name}`,
                error,
            );
            throw error;
        }
        this._name = definition.name;
        this._definition = definition;
    }

    get name() {
        return this._name;
    }

    get definition() {
        return this._definition;
    }

    async fetchRetry(url, options = {}, retries = 1, backoff = 300) {
        const retryCodes = [408, 500, 502, 503, 504, 522, 524];
        try {
            const res = await fetch(url, options);
            if (res.ok) {
                return res;
            }
            if (retries > 0 && retryCodes.includes(res.status)) {
                setTimeout(() => {
                    return this.fetchRetry(
                        url,
                        options,
                        retries - 1,
                        backoff * 2,
                    );
                }, backoff);
            }
            return res;
        } catch (error) {
            this.logger.warn(error);
            if (retries > 0) {
                setTimeout(() => {
                    return this.fetchRetry(
                        url,
                        options,
                        retries - 1,
                        backoff * 2,
                    );
                }, backoff);
            }
            return { status: 500 };
        }
    }
}

module.exports = Addon;
{'fraction_non_alphanumeric': 0.11651436031331593, 'fraction_numerical': 0.02121409921671018, 'mean_word_length': 1.3649691358024691, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6852351', 'n_tokens_mistral': 872, 'n_tokens_neox': 795, 'n_words': 194}