<commit_before>package org.unipop.schema.property;

import org.apache.commons.lang.NotImplementedException;
import org.apache.tinkerpop.gremlin.process.traversal.P;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer;
import org.json.JSONArray;
import org.json.JSONObject;
import org.unipop.query.predicates.PredicatesHolder;
import org.unipop.query.predicates.PredicatesHolderFactory;

import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;

/**
 * Created by sbarzilay on 7/28/16.
 */
public class MultiFieldPropertySchema implements PropertySchema {
    private String key;
    private List<String> fields;

    public MultiFieldPropertySchema(String key, List<String> fields) {
        this.key = key;
        this.fields = fields;
    }

    @Override
    public Map<String, Object> toProperties(Map<String, Object> source) {
        List<Object> value = new ArrayList<>();
        fields.forEach(field -> {
            if (field.startsWith("@"))
                value.add(source.get(field.substring(1)));
            else
                value.add(field);
        });
        return Collections.singletonMap(key, value);
    }

    @Override
    public Set<String> excludeDynamicFields() {
        return new HashSet<>(this.fields);
    }

    @Override
    public Set<String> excludeDynamicProperties() {
        return Collections.singleton(this.key);
    }

    @Override
    public Map<String, Object> toFields(Map<String, Object> properties) {
        return Collections.emptyMap();
    }

    @Override
    public Set<String> toFields(Set<String> propertyKeys) {
        return propertyKeys.contains(key) ? fields.stream()
                .filter(s -> s.startsWith("@")).map(s -> s.substring(1)).collect(Collectors.toSet()) :
                Collections.emptySet();
    }

    @Override
    public PredicatesHolder toPredicates(PredicatesHolder predicatesHolder) {
        Stream<HasContainer> hasContainers = predicatesHolder.findKey(this.key);

        Set<PredicatesHolder> predicateHolders = hasContainers.map(this::toPredicate).collect(Collectors.toSet());
        return PredicatesHolderFactory.create(predicatesHolder.getClause(), predicateHolders);
    }

    private PredicatesHolder toPredicate(HasContainer hasContainer) {
        Object value = hasContainer.getValue();
        Set<HasContainer> predicates = new HashSet<>();
        if (value instanceof Collection) {
            for (String field : fields) {
                if (field.startsWith("@")) {
                    for (Object v : ((Collection) value)) {
                        HasContainer clone = new HasContainer(field.substring(1), new P(hasContainer.getBiPredicate(), v));
                        predicates.add(clone);
                    }
                } else {
                    if (new P(hasContainer.getBiPredicate(), hasContainer.getValue()).test(field)) {
                        return PredicatesHolderFactory.empty();
                    }
                }
            }
        } else {
            final boolean[] abort = {false};
            fields.forEach(field -> {
                if (field.startsWith("@")) {
                    HasContainer clone = hasContainer.clone();
                    clone.setKey(field.substring(1));
                    predicates.add(clone);
                } else {
                    if (new P(hasContainer.getBiPredicate(), hasContainer.getValue()).test(field)) {
                        abort[0] = true;
                        return;
                    }
                }
            });
            if (abort[0])
                return PredicatesHolderFactory.empty();
        }
        return PredicatesHolderFactory.or(predicates.toArray(new HasContainer[predicates.size()]));
    }

    public static class Builder implements PropertySchemaBuilder {
        @Override
        public PropertySchema build(String key, Object conf) {
            if (!(conf instanceof JSONArray)) return null;
            JSONArray fieldsArray = (JSONArray) conf;
            List<String> fields = new ArrayList<>();
            for (int i = 0; i < fieldsArray.length(); i++) {
                String field = fieldsArray.getString(i);
                fields.add(field);
            }
            return new MultiFieldPropertySchema(key, fields);
        }
    }<commit_msg>converted MultiFieldPropertySchema to use child schemas<commit_after>package org.unipop.schema.property;

import org.apache.commons.lang.NotImplementedException;
import org.apache.tinkerpop.gremlin.process.traversal.P;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer;
import org.json.JSONArray;
import org.json.JSONObject;
import org.unipop.query.predicates.PredicatesHolder;
import org.unipop.query.predicates.PredicatesHolderFactory;

import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;

/**
 * Created by sbarzilay on 7/28/16.
 */
public class MultiFieldPropertySchema implements PropertySchema {
    private String key;
    private List<PropertySchema> schemas;

    public MultiFieldPropertySchema(String key, List<PropertySchema> schemas) {
        this.key = key;
        this.schemas = schemas;
    }

    @Override
    public Map<String, Object> toProperties(Map<String, Object> source) {
        List<Object> value = new ArrayList<>();
        schemas.forEach(schema -> {
            schema.toProperties(source).values().forEach(value::add);
        });
        return Collections.singletonMap(key, value);
    }

    @Override
    public Set<String> excludeDynamicFields() {
        return schemas.stream()
                .map(PropertySchema::excludeDynamicFields)
                .flatMap(Collection::stream).collect(Collectors.toSet());
    }

    @Override
    public Set<String> excludeDynamicProperties() {
        return Collections.singleton(this.key);
    }

    @Override
    public Map<String, Object> toFields(Map<String, Object> properties) {
        return Collections.emptyMap();
    }

    @Override
    public Set<String> toFields(Set<String> propertyKeys) {
        return propertyKeys.contains(key) ? schemas.stream()
                .flatMap(s -> s.toFields(propertyKeys).stream()).collect(Collectors.toSet()) :
                Collections.emptySet();
    }

    @Override
    public PredicatesHolder toPredicates(PredicatesHolder predicatesHolder) {
        Stream<HasContainer> hasContainers = predicatesHolder.findKey(this.key);

        Set<PredicatesHolder> predicateHolders = hasContainers.map(this::toPredicate).collect(Collectors.toSet());
        return PredicatesHolderFactory.create(predicatesHolder.getClause(), predicateHolders);
    }

    private PredicatesHolder toPredicate(HasContainer hasContainer) {
        Set<PredicatesHolder> predicates = new HashSet<>();
        for (PropertySchema schema : schemas) {
            PredicatesHolder predicatesHolder = schema.toPredicates(PredicatesHolderFactory.predicate(hasContainer));
            if (predicatesHolder.equals(PredicatesHolderFactory.empty()))
                return predicatesHolder;
            if (!predicatesHolder.equals(PredicatesHolderFactory.abort()))
                predicates.add(predicatesHolder);
        }
        return PredicatesHolderFactory.or(predicates);
    }

    public static class Builder implements PropertySchemaBuilder {
        @Override
        public PropertySchema build(String key, Object conf) {
            if (!(conf instanceof JSONArray)) return null;
            JSONArray fieldsArray = (JSONArray) conf;
            List<PropertySchema> schemas = new ArrayList<>();
            for (int i = 0; i < fieldsArray.length(); i++) {
                schemas.add(AbstractPropertyContainer.createPropertySchema(key, fieldsArray.get(i), true));
            }
            return new MultiFieldPropertySchema(key, schemas);
        }
    }
{'fraction_non_alphanumeric': 0.10655530245507466, 'fraction_numerical': 0.002277904328018223, 'mean_word_length': 2.589009990917348, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '420335', 'n_tokens_mistral': 2097, 'n_tokens_neox': 2019, 'n_words': 407}