<commit_before>module.exports = {
    config: {
        dryRun: false,
        gracePeriod: 0,
    },

    getAssets: function (space, assets = [], skip = 0, limit = 1000) {
        return space.getAssets({skip, limit}).then(response => {
            assets = assets.concat(response.items);
            if (response.total > skip + limit) {
                return this.getAssets(space, assets, skip + limit, limit);
            }
            return assets;
        });
    },

    getEntries: function (space, entries = [], skip = 0, limit = 1000) {
        return space.getEntries({skip, limit}).then(response => {
            entries = entries.concat(response.items);
            if (response.total > skip + limit) {
                return this.getEntries(space, entries, skip + limit, limit);
            }
            return entries;
        });
    },

    deleteEntity: function (entity) {
        if (this.config.isDryRun) {
            return;
        }

        if (entity.isPublished()) {
            return entity.unpublish().then(() => entity.delete());
        }

        return entity.delete();
    },

    getAgeInDays: function (entity) {
        const updatedAt = new Date(entity.sys.updatedAt);
        const age = (new Date() - updatedAt);

        return age / (24 * 60 * 60 * 1000);
    },

    isInGracePeriod: function (entity) {
        return this.getAgeInDays(entity) <= this.config.gracePeriod;
    }<commit_msg>adhere to API rate limits when deleting<commit_after>const Promise = require('sync-p');

const minDelay = 100;

let lastDeletion = +new Date();

const delay = function (time) {
    return new Promise(resolve => setTimeout(() => resolve(true), time));
};

const deleteEntity = function (entity) {
    const now = +new Date();

    if (now - lastDeletion > minDelay) {
        lastDeletion = now;
        return entity.delete();
    }

    lastDeletion += minDelay;
    return delay(lastDeletion - now).then(() => entity.delete());
};

module.exports = {
    config: {
        dryRun: false,
        gracePeriod: 0,
    },

    getAssets: function (space, assets = [], skip = 0, limit = 1000) {
        return space.getAssets({skip, limit}).then(response => {
            assets = assets.concat(response.items);
            if (response.total > skip + limit) {
                return this.getAssets(space, assets, skip + limit, limit);
            }
            return assets;
        });
    },

    getEntries: function (space, entries = [], skip = 0, limit = 1000) {
        return space.getEntries({skip, limit}).then(response => {
            entries = entries.concat(response.items);
            if (response.total > skip + limit) {
                return this.getEntries(space, entries, skip + limit, limit);
            }
            return entries;
        });
    },

    deleteEntity: function (entity) {
        if (this.config.isDryRun) {
            return;
        }

        if (entity.isPublished()) {
            return entity.unpublish().then(() => deleteEntity(entity));
        }

        return deleteEntity(entity);
    },

    getAgeInDays: function (entity) {
        const updatedAt = +new Date(entity.sys.updatedAt);
        const age = (+new Date() - updatedAt);

        return age / (24 * 60 * 60 * 1000);
    },

    isInGracePeriod: function (entity) {
        return this.getAgeInDays(entity) <= this.config.gracePeriod;
    }
{'fraction_non_alphanumeric': 0.13431516936671575, 'fraction_numerical': 0.013254786450662739, 'mean_word_length': 2.032142857142857, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5786171', 'n_tokens_mistral': 1016, 'n_tokens_neox': 936, 'n_words': 234}