<commit_before>package net.researchgate.restdsl.dao;

import com.google.common.base.Joiner;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import net.researchgate.restdsl.domain.EntityIndexInfo;
import net.researchgate.restdsl.domain.EntityInfo;
import net.researchgate.restdsl.exceptions.RestDslException;
import net.researchgate.restdsl.metrics.NoOpStatsReporter;
import net.researchgate.restdsl.metrics.StatsReporter;
import net.researchgate.restdsl.metrics.StatsTimingWrapper;
import net.researchgate.restdsl.queries.ServiceQuery;
import net.researchgate.restdsl.queries.ServiceQueryInfo;
import net.researchgate.restdsl.queries.ServiceQueryReservedValue;
import net.researchgate.restdsl.results.EntityList;
import net.researchgate.restdsl.results.EntityMultimap;
import net.researchgate.restdsl.results.EntityResult;
import net.researchgate.restdsl.types.TypeInfoUtil;
import net.researchgate.restdsl.util.ServiceQueryUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.mongodb.morphia.Datastore;
import org.mongodb.morphia.dao.BasicDAO;
import org.mongodb.morphia.query.Query;
import org.mongodb.morphia.query.UpdateOperations;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * This Dao implements read only access to the underlying mongo collection.
 * Use this dao if you want to make sure that your data is only written to in a controlled way.
 *
 * If you want to simply expose CRUD via REST, use a {@link MongoServiceDao}.
 *
 * @param <V> Type of the entity
 * @param <K> Type of the entity's id field
 */
public class BaseMongoServiceDao<V, K> implements ServiceDao<V, K>{
    private static final Logger LOGGER = LoggerFactory.getLogger(BaseMongoServiceDao.class);

    private static final String QUERY_KEY = "queries.shapes.%s.%%H";
    protected final String collectionName;
    protected final BasicDAO<V, K> morphiaDao;
    protected final Class<V> entityClazz;
    protected final EntityInfo<V> entityInfo;
    protected final EntityIndexInfo<V> entityIndexInfo;
    protected final StatsReporter statsReporter;

    // group by operations may require a lot requests to the database. We should have to explicitly enable it
    protected boolean allowGroupBy = false;

    public BaseMongoServiceDao(Datastore datastore, Class<V> entityClazz) {
        this(datastore, entityClazz, NoOpStatsReporter.INSTANCE);
    }
    //TODO: provide implementations for StatsReporter in example service
    public BaseMongoServiceDao(Datastore datastore, Class<V> entityClazz, StatsReporter statsReporter) {
        this.morphiaDao = new BasicDAO<>(entityClazz, datastore);
        this.collectionName = morphiaDao.getCollection().getName();
        this.entityClazz = entityClazz;
        this.entityInfo = EntityInfo.get(entityClazz);
        this.entityIndexInfo = new EntityIndexInfo<>(entityClazz, morphiaDao.getCollection().getIndexInfo());
        this.statsReporter = statsReporter;
    }

    /**
     * explicitly allow groupBy in get queries.
     *
     * If not, we will throw a RestDsl exception when the client queries with a groupBy.
     */
    protected void setAllowGroupBy() {
        this.allowGroupBy = true;
    }

    public EntityResult<V> get(ServiceQuery<K> serviceQuery) throws RestDslException {
        Query<V> morphiaQuery = convertToMorphiaQuery(serviceQuery);

        LOGGER.debug("Executing query {}", morphiaQuery);

        try (StatsTimingWrapper ignored = getQueryShapeWrapper(serviceQuery)) {
            String groupBy = serviceQuery.getGroupBy();
            if (groupBy == null) {
                List<V> results = Collections.emptyList();
                if (!serviceQuery.getCountOnly()) {
                    results = morphiaDao.find(morphiaQuery).asList();
                }
                return new EntityResult<>(results, getTotalItemsCnt(morphiaQuery, serviceQuery, results));
            } else {
                if (!allowGroupBy) {
                    throw new RestDslException("GroupBy is not allowed by this dao, but rquest contains groupBy '" + groupBy + "'. GroupBy can be enabled in the Service", RestDslException.Type.QUERY_ERROR);
                }
                // warning: in-place criteria editing
                Collection<Object> criteriaForGrouping = Lists.newArrayList(serviceQuery.getCriteria().get(groupBy));

                Map<Object, EntityList<V>> groupedResult = new HashMap<>();
                for (Object k : criteriaForGrouping) {
                    serviceQuery.getCriteria().removeAll(groupBy);
                    serviceQuery.getCriteria().put(groupBy, k);
                    Query<V> q = convertToMorphiaQuery(serviceQuery);
                    List<V> resultPerKey = Collections.emptyList();
                    if (!serviceQuery.getCountOnly()) {
                        resultPerKey = morphiaDao.find(q).asList();
                    }

                    EntityList<V> entityList = new EntityList<>(resultPerKey, getTotalItemsCnt(q, serviceQuery, resultPerKey));
                    groupedResult.put(k, entityList);
                }
                return new EntityResult<>(new EntityMultimap<>(groupedResult, serviceQuery.isCountTotalItems() ? morphiaQuery.count() : null));
            }
        }
    }

    public V getOne(ServiceQuery<K> serviceQuery) throws RestDslException {
        return morphiaDao.findOne(convertToMorphiaQuery(serviceQuery));
    }

    public long count(ServiceQuery<K> serviceQuery) throws RestDslException {
        return convertToMorphiaQuery(serviceQuery).count();
    }

    protected UpdateOperations<V> createUpdateOperations() {
        return morphiaDao.createUpdateOperations();
    }

    protected Query<V> convertToMorphiaQuery(ServiceQuery<K> serviceQuery) throws RestDslException {
        return convertToMorphiaQuery(serviceQuery, true);
    }

    protected Query<V> convertToMorphiaQuery(ServiceQuery<K> serviceQuery, boolean getQuery) throws RestDslException {
        validateQuery(serviceQuery);

        Query<V> mongoQuery = morphiaDao.createQuery();

        Collection<K> ids = serviceQuery.getIdList();
        if (ids != null) {
            if (ids.size() == 1) {
                mongoQuery.field(entityInfo.getIdFieldName()).equal(ids.iterator().next());
            } else {
                mongoQuery.field(entityInfo.getIdFieldName()).hasAnyOf(ids);
            }
        }<commit_msg>implement delete in mongodao<commit_after>package net.researchgate.restdsl.dao;

import com.google.common.base.Joiner;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import net.researchgate.restdsl.domain.EntityIndexInfo;
import net.researchgate.restdsl.domain.EntityInfo;
import net.researchgate.restdsl.exceptions.RestDslException;
import net.researchgate.restdsl.metrics.NoOpStatsReporter;
import net.researchgate.restdsl.metrics.StatsReporter;
import net.researchgate.restdsl.metrics.StatsTimingWrapper;
import net.researchgate.restdsl.queries.ServiceQuery;
import net.researchgate.restdsl.queries.ServiceQueryInfo;
import net.researchgate.restdsl.queries.ServiceQueryReservedValue;
import net.researchgate.restdsl.results.EntityList;
import net.researchgate.restdsl.results.EntityMultimap;
import net.researchgate.restdsl.results.EntityResult;
import net.researchgate.restdsl.types.TypeInfoUtil;
import net.researchgate.restdsl.util.ServiceQueryUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.mongodb.morphia.Datastore;
import org.mongodb.morphia.dao.BasicDAO;
import org.mongodb.morphia.query.Query;
import org.mongodb.morphia.query.UpdateOperations;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
 * This Dao implements common access to the underlying mongo collection.
 * Use this dao if you want to make sure that your data is only written to in a controlled way.
 *
 * Supported operations
 * - get by restle dsl
 * - delete by id
 *
 * If you want to simply expose CRUD via REST, use a {@link MongoServiceDao}.
 *
 * @param <V> Type of the entity
 * @param <K> Type of the entity's id field
 */
public class BaseMongoServiceDao<V, K> implements ServiceDao<V, K>{
    private static final Logger LOGGER = LoggerFactory.getLogger(BaseMongoServiceDao.class);

    private static final String QUERY_KEY = "queries.shapes.%s.%%H";
    protected final String collectionName;
    protected final BasicDAO<V, K> morphiaDao;
    protected final Class<V> entityClazz;
    protected final EntityInfo<V> entityInfo;
    protected final EntityIndexInfo<V> entityIndexInfo;
    protected final StatsReporter statsReporter;

    // group by operations may require a lot requests to the database. We should have to explicitly enable it
    protected boolean allowGroupBy = false;

    public BaseMongoServiceDao(Datastore datastore, Class<V> entityClazz) {
        this(datastore, entityClazz, NoOpStatsReporter.INSTANCE);
    }
    //TODO: provide implementations for StatsReporter in example service
    public BaseMongoServiceDao(Datastore datastore, Class<V> entityClazz, StatsReporter statsReporter) {
        this.morphiaDao = new BasicDAO<>(entityClazz, datastore);
        this.collectionName = morphiaDao.getCollection().getName();
        this.entityClazz = entityClazz;
        this.entityInfo = EntityInfo.get(entityClazz);
        this.entityIndexInfo = new EntityIndexInfo<>(entityClazz, morphiaDao.getCollection().getIndexInfo());
        this.statsReporter = statsReporter;
    }

    /**
     * explicitly allow groupBy in get queries.
     *
     * If not, we will throw a RestDsl exception when the client queries with a groupBy.
     */
    protected void setAllowGroupBy() {
        this.allowGroupBy = true;
    }

    public EntityResult<V> get(ServiceQuery<K> serviceQuery) throws RestDslException {
        Query<V> morphiaQuery = convertToMorphiaQuery(serviceQuery);

        LOGGER.debug("Executing query {}", morphiaQuery);

        try (StatsTimingWrapper ignored = getQueryShapeWrapper(serviceQuery)) {
            String groupBy = serviceQuery.getGroupBy();
            if (groupBy == null) {
                List<V> results = Collections.emptyList();
                if (!serviceQuery.getCountOnly()) {
                    results = morphiaDao.find(morphiaQuery).asList();
                }
                return new EntityResult<>(results, getTotalItemsCnt(morphiaQuery, serviceQuery, results));
            } else {
                if (!allowGroupBy) {
                    throw new RestDslException("GroupBy is not allowed by this dao, but rquest contains groupBy '" + groupBy + "'. GroupBy can be enabled in the Service", RestDslException.Type.QUERY_ERROR);
                }
                // warning: in-place criteria editing
                Collection<Object> criteriaForGrouping = Lists.newArrayList(serviceQuery.getCriteria().get(groupBy));

                Map<Object, EntityList<V>> groupedResult = new HashMap<>();
                for (Object k : criteriaForGrouping) {
                    serviceQuery.getCriteria().removeAll(groupBy);
                    serviceQuery.getCriteria().put(groupBy, k);
                    Query<V> q = convertToMorphiaQuery(serviceQuery);
                    List<V> resultPerKey = Collections.emptyList();
                    if (!serviceQuery.getCountOnly()) {
                        resultPerKey = morphiaDao.find(q).asList();
                    }

                    EntityList<V> entityList = new EntityList<>(resultPerKey, getTotalItemsCnt(q, serviceQuery, resultPerKey));
                    groupedResult.put(k, entityList);
                }
                return new EntityResult<>(new EntityMultimap<>(groupedResult, serviceQuery.isCountTotalItems() ? morphiaQuery.count() : null));
            }
        }
    }

    public V getOne(ServiceQuery<K> serviceQuery) throws RestDslException {
        return morphiaDao.findOne(convertToMorphiaQuery(serviceQuery));
    }

    public long count(ServiceQuery<K> serviceQuery) throws RestDslException {
        return convertToMorphiaQuery(serviceQuery).count();
    }

    /**
     * Delete the entity by id value
     * @param id the ID of the document to delete
     * @return the number of deleted items (0 or 1)
     */
    public int delete(K id) {
        return morphiaDao.deleteById(id).getN();

    }

    protected UpdateOperations<V> createUpdateOperations() {
        return morphiaDao.createUpdateOperations();
    }

    protected Query<V> convertToMorphiaQuery(ServiceQuery<K> serviceQuery) throws RestDslException {
        return convertToMorphiaQuery(serviceQuery, true);
    }

    protected Query<V> convertToMorphiaQuery(ServiceQuery<K> serviceQuery, boolean getQuery) throws RestDslException {
        validateQuery(serviceQuery);

        Query<V> mongoQuery = morphiaDao.createQuery();

        Collection<K> ids = serviceQuery.getIdList();
        if (ids != null) {
            if (ids.size() == 1) {
                mongoQuery.field(entityInfo.getIdFieldName()).equal(ids.iterator().next());
            } else {
                mongoQuery.field(entityInfo.getIdFieldName()).hasAnyOf(ids);
            }
        }
{'fraction_non_alphanumeric': 0.09438218599912676, 'fraction_numerical': 0.0007276961141027507, 'mean_word_length': 3.2693383038210624, 'pattern_counts': {'":': 0, '<': 75, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2896285', 'n_tokens_mistral': 3907, 'n_tokens_neox': 3832, 'n_words': 873}