<commit_before>package io.cattle.platform.object.purge.impl;

import io.cattle.platform.archaius.util.ArchaiusUtil;
import io.cattle.platform.engine.manager.ProcessManager;
import io.cattle.platform.engine.manager.ProcessNotFoundException;
import io.cattle.platform.engine.process.ProcessDefinition;
import io.cattle.platform.engine.process.ProcessInstanceException;
import io.cattle.platform.object.ObjectManager;
import io.cattle.platform.object.meta.ObjectMetaDataManager;
import io.cattle.platform.object.process.ObjectProcessManager;
import io.cattle.platform.object.process.StandardProcess;
import io.cattle.platform.object.purge.RemoveMonitor;
import io.cattle.platform.object.util.ObjectUtils;
import io.cattle.platform.task.Task;
import io.github.ibuildthecloud.gdapi.condition.Condition;
import io.github.ibuildthecloud.gdapi.condition.ConditionType;
import io.github.ibuildthecloud.gdapi.factory.SchemaFactory;
import io.github.ibuildthecloud.gdapi.model.Schema;

import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import javax.inject.Inject;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.netflix.config.DynamicLongProperty;

public class RemoveMonitorImpl implements RemoveMonitor, Task {

    @Inject
    SchemaFactory schemaFactory;

    @Inject
    ObjectMetaDataManager objectMetaDataManager;

    @Inject
    ObjectProcessManager objectProcessManager;

    @Inject
    ProcessManager processManager;

    @Inject
    ObjectManager objectManager;

    private static final String ERROR_STATE = "error";
    private static final Logger log = LoggerFactory.getLogger(RemoveMonitorImpl.class);
    private static final DynamicLongProperty REMOVE_DELAY = ArchaiusUtil.getLong("remove.resources.after");


    @Override
    public void run() {
        for (String type : findRemovableTypes()) {
            Class<?> schemaClass = schemaFactory.getSchemaClass(type);
            if (schemaClass == null) {
                continue;
            }

            List<?> objects = objectManager.find(schemaClass, ObjectMetaDataManager.STATE_FIELD, ERROR_STATE,
                    ObjectMetaDataManager.CREATED_FIELD,
                    new Condition(ConditionType.NOTNULL), ObjectMetaDataManager.CREATED_FIELD, new Condition(
                            ConditionType.LT, new Date(System.currentTimeMillis() - REMOVE_DELAY.get() * 1000)));

            for (Object obj : objects) {
                try {
                    objectProcessManager.scheduleStandardProcess(StandardProcess.REMOVE, obj, null);
                    log.info("Scheduling remove for [{}] id [{}]", type, ObjectUtils.getId(obj));
                } catch (ProcessNotFoundException e) {
                } catch (ProcessInstanceException e) {
                    log.info("Failed to scheduling remove for [{}] id [{}]", type, ObjectUtils.getId(obj), e);
                }
            }
        }
    }

    @Override
    public String getName() {
        return "remove.resources";
    }

    protected synchronized Set<String> findRemovableTypes() {
        Set<String> types = new HashSet<String>();
        for (Schema schema : schemaFactory.listSchemas()) {
            while (schema.getParent() != null) {
                schema = schemaFactory.getSchema(schema.getParent());
            }

            String type = schema.getId();

            if (types.contains(type)) {
                continue;
            }

            Object stateField = objectMetaDataManager.convertFieldNameFor(type, ObjectMetaDataManager.STATE_FIELD);
            if (stateField == null) {
                continue;
            }

            String processName = objectProcessManager.getStandardProcessName(StandardProcess.REMOVE, type);
            ProcessDefinition def = processManager.getProcessDefinition(processName);

            if (def != null) {
                types.add(type);
            }
        }

        return types;
    }
}
<commit_msg>Restrict error resource removal to instance<commit_after>package io.cattle.platform.object.purge.impl;

import io.cattle.platform.archaius.util.ArchaiusUtil;
import io.cattle.platform.engine.manager.ProcessManager;
import io.cattle.platform.engine.manager.ProcessNotFoundException;
import io.cattle.platform.engine.process.ProcessDefinition;
import io.cattle.platform.engine.process.ProcessInstanceException;
import io.cattle.platform.object.ObjectManager;
import io.cattle.platform.object.meta.ObjectMetaDataManager;
import io.cattle.platform.object.process.ObjectProcessManager;
import io.cattle.platform.object.process.StandardProcess;
import io.cattle.platform.object.purge.RemoveMonitor;
import io.cattle.platform.object.util.ObjectUtils;
import io.cattle.platform.task.Task;
import io.github.ibuildthecloud.gdapi.condition.Condition;
import io.github.ibuildthecloud.gdapi.condition.ConditionType;
import io.github.ibuildthecloud.gdapi.factory.SchemaFactory;
import io.github.ibuildthecloud.gdapi.model.Schema;

import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import javax.inject.Inject;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.netflix.config.DynamicLongProperty;

public class RemoveMonitorImpl implements RemoveMonitor, Task {

    @Inject
    SchemaFactory schemaFactory;

    @Inject
    ObjectMetaDataManager objectMetaDataManager;

    @Inject
    ObjectProcessManager objectProcessManager;

    @Inject
    ProcessManager processManager;

    @Inject
    ObjectManager objectManager;

    private static final String ERROR_STATE = "error";
    private static final Logger log = LoggerFactory.getLogger(RemoveMonitorImpl.class);
    private static final DynamicLongProperty REMOVE_DELAY = ArchaiusUtil.getLong("remove.resources.after");


    @Override
    public void run() {
        for (String type : findRemovableTypes()) {
            Class<?> schemaClass = schemaFactory.getSchemaClass(type);
            if (schemaClass == null) {
                continue;
            }

            List<?> objects = objectManager.find(schemaClass, ObjectMetaDataManager.STATE_FIELD, ERROR_STATE,
                    ObjectMetaDataManager.CREATED_FIELD,
                    new Condition(ConditionType.NOTNULL), ObjectMetaDataManager.CREATED_FIELD, new Condition(
                            ConditionType.LT, new Date(System.currentTimeMillis() - REMOVE_DELAY.get() * 1000)));

            for (Object obj : objects) {
                try {
                    objectProcessManager.scheduleStandardProcess(StandardProcess.REMOVE, obj, null);
                    log.info("Scheduling remove for [{}] id [{}]", type, ObjectUtils.getId(obj));
                } catch (ProcessNotFoundException e) {
                } catch (ProcessInstanceException e) {
                    log.info("Failed to scheduling remove for [{}] id [{}]", type, ObjectUtils.getId(obj), e);
                }
            }
        }
    }

    @Override
    public String getName() {
        return "remove.resources";
    }

    protected synchronized Set<String> findRemovableTypes() {
        Set<String> types = new HashSet<String>();
        for (Schema schema : schemaFactory.listSchemas()) {
            while (schema.getParent() != null) {
                schema = schemaFactory.getSchema(schema.getParent());
            }

            String type = schema.getId();

            if (types.contains(type)) {
                continue;
            }

            // restricting it to instance for now
            if (!type.equalsIgnoreCase("instance")) {
                continue;
            }

            Object stateField = objectMetaDataManager.convertFieldNameFor(type, ObjectMetaDataManager.STATE_FIELD);
            if (stateField == null) {
                continue;
            }

            String processName = objectProcessManager.getStandardProcessName(StandardProcess.REMOVE, type);
            ProcessDefinition def = processManager.getProcessDefinition(processName);

            if (def != null) {
                types.add(type);
            }
        }

        return types;
    }
}

{'fraction_non_alphanumeric': 0.09485826481776904, 'fraction_numerical': 0.0014725733218799854, 'mean_word_length': 2.787174721189591, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7155662', 'n_tokens_mistral': 2153, 'n_tokens_neox': 2107, 'n_words': 412}