<commit_before>package net.ripe.db.whois.common.rpsl;

import com.google.common.base.Splitter;
import com.google.common.collect.Sets;

import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;

// Helper class for less-frequently used information extractions from RpslObject
public class RpslObjectFilter {
    static final String FILTERED = " # Filtered";

    private static final Splitter LINE_CONTINUATION_SPLITTER = Splitter.on(Pattern.compile("(\\n\\+|\\n[ ]|\\n\\t|\\n)")).trimResults();

    private RpslObjectFilter() {
    }

    public static String getCertificateFromKeyCert(final RpslObject object) {
        if (!ObjectType.KEY_CERT.equals(object.getType())) {
            throw new AuthenticationException("No key cert for object: " + object.getKey());
        }

        final StringBuilder builder = new StringBuilder();
        for (RpslAttribute next : object.findAttributes(AttributeType.CERTIF)) {
            for (String s : LINE_CONTINUATION_SPLITTER.split(next.getValue())) {
                builder.append(s).append('\n');
            }
        }

        return builder.toString();
    }

    public static RpslObjectBuilder keepKeyAttributesOnly(RpslObjectBuilder builder) {
        final ObjectTemplate template = ObjectTemplate.getTemplate(ObjectType.getByFirstAttribute(builder.getTypeAttribute()));
        final Set<AttributeType> keyAttributes = Sets.newHashSet();
        keyAttributes.addAll(template.getLookupAttributes());
        keyAttributes.addAll(template.getKeyAttributes());
        return builder.retainAttributeTypes(keyAttributes);
    }

    public static RpslObjectBuilder setFiltered(RpslObjectBuilder builder) {
        for (int i = builder.size() - 1; i >= 0; i--) {
            RpslAttribute attribute = builder.getAttribute(i);
            if (attribute.getType() == AttributeType.SOURCE) {
                builder.setAttribute(i, new RpslAttribute(AttributeType.SOURCE, attribute.getCleanValue() + FILTERED));
                break;
            }
        }
        return builder;
    }

    public static void addFilteredSourceReplacement(final RpslObject object, final Map<RpslAttribute, RpslAttribute> replacementsMap) {
        RpslAttribute attribute = object.findAttribute(AttributeType.SOURCE);
        replacementsMap.put(attribute, new RpslAttribute(AttributeType.SOURCE, attribute.getCleanValue() + FILTERED));
    }

    public static boolean isFiltered(final RpslObject rpslObject) {
        final List<RpslAttribute> attributes = rpslObject.findAttributes(AttributeType.SOURCE);
        return !attributes.isEmpty() && attributes.get(0).getValue().contains(FILTERED);
    }
}<commit_msg>finished adding test coverage to x-forwarded-for passing<commit_after>package net.ripe.db.whois.common.rpsl;

import com.google.common.base.Splitter;
import com.google.common.collect.Sets;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;

// Helper class for less-frequently used information extractions from RpslObject
public class RpslObjectFilter {
    static final String FILTERED = " # Filtered";

    private static final Splitter LINE_CONTINUATION_SPLITTER = Splitter.on(Pattern.compile("(\\n\\+|\\n[ ]|\\n\\t|\\n)")).trimResults();

    private RpslObjectFilter() {
    }

    public static String getCertificateFromKeyCert(final RpslObject object) {
        if (!ObjectType.KEY_CERT.equals(object.getType())) {
            throw new AuthenticationException("No key cert for object: " + object.getKey());
        }

        final StringBuilder builder = new StringBuilder();
        for (RpslAttribute next : object.findAttributes(AttributeType.CERTIF)) {
            for (String s : LINE_CONTINUATION_SPLITTER.split(next.getValue())) {
                builder.append(s).append('\n');
            }
        }

        return builder.toString();
    }

    public static RpslObjectBuilder keepKeyAttributesOnly(RpslObjectBuilder builder) {
        final ObjectTemplate template = ObjectTemplate.getTemplate(ObjectType.getByFirstAttribute(builder.getTypeAttribute()));
        final Set<AttributeType> keyAttributes = Sets.newHashSet();
        keyAttributes.addAll(template.getLookupAttributes());
        keyAttributes.addAll(template.getKeyAttributes());
        return builder.retainAttributeTypes(keyAttributes);
    }

    public static RpslObjectBuilder setFiltered(RpslObjectBuilder builder) {
        for (int i = builder.size() - 1; i >= 0; i--) {
            RpslAttribute attribute = builder.getAttribute(i);
            if (attribute.getType() == AttributeType.SOURCE) {
                builder.setAttribute(i, new RpslAttribute(AttributeType.SOURCE, attribute.getCleanValue() + FILTERED));
                break;
            }
        }
        return builder;
    }

    public static void addFilteredSourceReplacement(final RpslObject object, final Map<RpslAttribute, RpslAttribute> replacementsMap) {
        RpslAttribute attribute = object.findAttribute(AttributeType.SOURCE);
        replacementsMap.put(attribute, new RpslAttribute(AttributeType.SOURCE, attribute.getCleanValue() + FILTERED));
    }

    public static boolean isFiltered(final RpslObject rpslObject) {
        final List<RpslAttribute> attributes = rpslObject.findAttributes(AttributeType.SOURCE);
        return !attributes.isEmpty() && attributes.get(0).getValue().contains(FILTERED);
    }

    /** slow way to build specific objects from object skeletons/templates. Meant for testing only */
    public static RpslObject buildGenericObject(final RpslObject object, final String ... attributes) {
        return buildGenericObject(new RpslObjectBuilder(object), attributes);
    }

    /** slow way to build specific objects from object skeletons/templates. Meant for testing only */
    public static RpslObject buildGenericObject(final String object, final String ... attributes) {
        return buildGenericObject(new RpslObjectBuilder(object), attributes);
    }

    /** slow way to build specific objects from object skeletons/templates. Meant for testing only */
    public static RpslObject buildGenericObject(final RpslObjectBuilder builder, final String ... attributes) {
        List<RpslAttribute> attributeList = new ArrayList<>();
        for (String attribute : attributes) {
            attributeList.addAll(RpslObjectBuilder.getAttributes(attribute));
        }
        for (RpslAttribute rpslAttribute : attributeList) {
            builder.removeAttributeType(rpslAttribute.getType());
        }

        builder.addAttributes(attributeList);
        return builder.sort().get();
    }

}
{'fraction_non_alphanumeric': 0.10164670658682634, 'fraction_numerical': 0.0008982035928143712, 'mean_word_length': 3.4451097804391217, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3436023', 'n_tokens_mistral': 1823, 'n_tokens_neox': 1781, 'n_words': 393}