<commit_before>package org.oasis.datacore.core.entity.query.ldp;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.oasis.datacore.core.entity.model.DCEntity;
import org.oasis.datacore.core.entity.mongodb.DatacoreMongoTemplate;
import org.oasis.datacore.core.entity.query.QueryException;
import org.oasis.datacore.core.meta.model.DCField;
import org.oasis.datacore.core.meta.model.DCMapField;
import org.oasis.datacore.core.meta.model.DCModel;
import org.oasis.datacore.core.meta.model.DCSecurity;
import org.oasis.datacore.core.security.DCUserImpl;
import org.oasis.datacore.core.security.service.DatacoreSecurityService;
import org.oasis.datacore.rest.server.MonitoringLogServiceImpl;
import org.oasis.datacore.rest.server.event.EventService;
import org.oasis.datacore.rest.server.parsing.model.DCQueryParsingContext;
import org.oasis.datacore.rest.server.parsing.model.DCResourceParsingContext;
import org.oasis.datacore.rest.server.parsing.service.QueryParsingService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.Sort;
import org.springframework.data.domain.Sort.Direction;
import org.springframework.data.mongodb.core.CursorProviderQueryCursorPreparer;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Component;

import com.mongodb.DBObject;


/**
 * Provides W3C LDP (Linked Data Platform)-like query on top of Datacore MongoDB storage.
 * 
 * TODO LATER move to -core : by removing parsing (& dependency to queryParsingContext)
 * & temporary dependency to DatacoreApiImpl, or by making -core depend on -server...
 * 
 * @author mdutoo
 *
 */
@Component
public class LdpEntityQueryServiceImpl implements LdpEntityQueryService {

   private static final Logger logger = LoggerFactory.getLogger(EventService.class);
   
   private static Set<String> findConfParams = new HashSet<String>();
   static {
      // TODO rather using Enum, see BSON$RegexFlag
      findConfParams.add("start");
      findConfParams.add("limit");
   }
   private static Map<String,DCField> dcEntityIndexedFields = new HashMap<String,DCField>();
   static {
      // TODO rather using Enum, see BSON$RegexFlag
      dcEntityIndexedFields.put("_uri", new DCField("_uri", "string", true, 100000));
      //dcEntityIndexedFields.add("_allReaders"); // don't allow to look it up
      dcEntityIndexedFields.put("_changedAt", new DCField("_chAt", "date", true, 100000));
   }

   /** default maximum number of documents to scan when fulfilling a query, overriden by
    * DCFields', themselves limited by DCModel's. 0 means no limit (for tests), else ex.
    * 1000 (secure default), 100000 (on query-only nodes using secondary & timeout)... 
    * http://docs.mongodb.org/manual/reference/operator/meta/maxScan/ */
   @Value("${datacoreApiServer.query.maxScan}")
   private int maxScan;
   /** default maximum start position : 500... */
   @Value("${datacoreApiServer.query.maxStart}")
   private int maxStart;
   /** default maximum number of documents returned : 100... */
   @Value("${datacoreApiServer.query.maxLimit}")
   private int maxLimit;
   
   @Autowired
   @Qualifier("datacoreSecurityServiceImpl")
   private DatacoreSecurityService datacoreSecurityService;
   
   @Autowired
   private /*MongoOperations*/DatacoreMongoTemplate mgo; // TODO remove it by hiding it in services
   
   @Autowired
   private QueryParsingService queryParsingService;

   @Autowired
   private MonitoringLogServiceImpl monitoringLogServiceImpl;

   
   @Override
   public List<DCEntity> findDataInType(DCModel dcModel, Map<String, List<String>> params,
         Integer start, Integer limit) throws QueryException {
      boolean detailedErrorsMode = true; // TODO

      //Log to AuditLog Endpoint
      //monitoringLogServiceImpl.postLog(modelType, "findDataInType");

      // parsing query parameters criteria according to model :
      DCQueryParsingContext queryParsingContext = new DCQueryParsingContext(dcModel, null);
      parseQueryParameters(params, queryParsingContext);
      if (queryParsingContext.hasErrors()) {
         String msg = DCResourceParsingContext.formatParsingErrorsMessage(queryParsingContext, detailedErrorsMode);
         throw new QueryException(msg);
      } // else TODO if warnings return them as response header ?? or only if failIfWarningsMode ??
      
      // add security :
      boolean guestForbidden = addSecurityIfNotGuestForbidden(queryParsingContext);
      if (guestForbidden) {
         return new ArrayList<DCEntity>(0); // TODO or exception ??
      }
      
      // adding paging & sorting :
      if (start > 500) {
         start = maxStart; // max, else prefer ranged query ; TODO or error message ?
      }
      if (limit > 50) {
         limit = maxLimit; // max, else prefer ranged query ; TODO or error message ?
      }
      Sort sort = queryParsingContext.getSort();
      if (sort == null) {
         // TODO sort by default : configured in model (last modified date, uri,
         // iri?, types?, owners? other fields...)
         sort = new Sort(Direction.ASC, "_chAt"); // _uri...
      }
      
      Query springMongoQuery = new Query(queryParsingContext.getCriteria())
         .with(sort).skip(start).limit(limit); // TODO rather range query, if possible on sort field
      
      List<DCEntity> foundEntities = executeMongoDbQuery(springMongoQuery, queryParsingContext);
      
      if (logger.isDebugEnabled()) {
         logger.debug("Done Spring Mongo query: " + springMongoQuery
               + "\n   in collection " + dcModel.getCollectionName()
               + " from Model " + dcModel.getName() + " and parameters: " + params
               + "\n   with result nb " + foundEntities.size());
      }
      return foundEntities;
   }

   
   private void parseQueryParameters(Map<String, List<String>> params, DCQueryParsingContext queryParsingContext) {
      DCModel dcModel = queryParsingContext.peekModel();
      
      parameterLoop : for (String fieldPath : params.keySet()) {
         if (findConfParams.contains(fieldPath)) {
            // skip find conf params
            continue;
         }
         
         //DCField dcField = dcModelService.getFieldByPath(dcModel, fieldPath); // TODO TODOOOOOOOOOOOOO<commit_msg>query sorted desc by modified<commit_after>package org.oasis.datacore.core.entity.query.ldp;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.oasis.datacore.core.entity.model.DCEntity;
import org.oasis.datacore.core.entity.mongodb.DatacoreMongoTemplate;
import org.oasis.datacore.core.entity.query.QueryException;
import org.oasis.datacore.core.meta.model.DCField;
import org.oasis.datacore.core.meta.model.DCMapField;
import org.oasis.datacore.core.meta.model.DCModel;
import org.oasis.datacore.core.meta.model.DCSecurity;
import org.oasis.datacore.core.security.DCUserImpl;
import org.oasis.datacore.core.security.service.DatacoreSecurityService;
import org.oasis.datacore.rest.server.MonitoringLogServiceImpl;
import org.oasis.datacore.rest.server.event.EventService;
import org.oasis.datacore.rest.server.parsing.model.DCQueryParsingContext;
import org.oasis.datacore.rest.server.parsing.model.DCResourceParsingContext;
import org.oasis.datacore.rest.server.parsing.service.QueryParsingService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.Sort;
import org.springframework.data.domain.Sort.Direction;
import org.springframework.data.mongodb.core.CursorProviderQueryCursorPreparer;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Component;

import com.mongodb.DBObject;


/**
 * Provides W3C LDP (Linked Data Platform)-like query on top of Datacore MongoDB storage.
 * 
 * TODO LATER move to -core : by removing parsing (& dependency to queryParsingContext)
 * & temporary dependency to DatacoreApiImpl, or by making -core depend on -server...
 * 
 * @author mdutoo
 *
 */
@Component
public class LdpEntityQueryServiceImpl implements LdpEntityQueryService {

   private static final Logger logger = LoggerFactory.getLogger(EventService.class);
   
   private static Set<String> findConfParams = new HashSet<String>();
   static {
      // TODO rather using Enum, see BSON$RegexFlag
      findConfParams.add("start");
      findConfParams.add("limit");
   }
   private static Map<String,DCField> dcEntityIndexedFields = new HashMap<String,DCField>();
   static {
      // TODO rather using Enum, see BSON$RegexFlag
      dcEntityIndexedFields.put("_uri", new DCField("_uri", "string", true, 100000));
      //dcEntityIndexedFields.add("_allReaders"); // don't allow to look it up
      dcEntityIndexedFields.put("_changedAt", new DCField("_chAt", "date", true, 100000));
   }

   /** default maximum number of documents to scan when fulfilling a query, overriden by
    * DCFields', themselves limited by DCModel's. 0 means no limit (for tests), else ex.
    * 1000 (secure default), 100000 (on query-only nodes using secondary & timeout)... 
    * http://docs.mongodb.org/manual/reference/operator/meta/maxScan/ */
   @Value("${datacoreApiServer.query.maxScan}")
   private int maxScan;
   /** default maximum start position : 500... */
   @Value("${datacoreApiServer.query.maxStart}")
   private int maxStart;
   /** default maximum number of documents returned : 100... */
   @Value("${datacoreApiServer.query.maxLimit}")
   private int maxLimit;
   
   @Autowired
   @Qualifier("datacoreSecurityServiceImpl")
   private DatacoreSecurityService datacoreSecurityService;
   
   @Autowired
   private /*MongoOperations*/DatacoreMongoTemplate mgo; // TODO remove it by hiding it in services
   
   @Autowired
   private QueryParsingService queryParsingService;

   @Autowired
   private MonitoringLogServiceImpl monitoringLogServiceImpl;

   
   @Override
   public List<DCEntity> findDataInType(DCModel dcModel, Map<String, List<String>> params,
         Integer start, Integer limit) throws QueryException {
      boolean detailedErrorsMode = true; // TODO

      //Log to AuditLog Endpoint
      //monitoringLogServiceImpl.postLog(modelType, "findDataInType");

      // parsing query parameters criteria according to model :
      DCQueryParsingContext queryParsingContext = new DCQueryParsingContext(dcModel, null);
      parseQueryParameters(params, queryParsingContext);
      if (queryParsingContext.hasErrors()) {
         String msg = DCResourceParsingContext.formatParsingErrorsMessage(queryParsingContext, detailedErrorsMode);
         throw new QueryException(msg);
      } // else TODO if warnings return them as response header ?? or only if failIfWarningsMode ??
      
      // add security :
      boolean guestForbidden = addSecurityIfNotGuestForbidden(queryParsingContext);
      if (guestForbidden) {
         return new ArrayList<DCEntity>(0); // TODO or exception ??
      }
      
      // adding paging & sorting :
      if (start > 500) {
         start = maxStart; // max, else prefer ranged query ; TODO or error message ?
      }
      if (limit > 50) {
         limit = maxLimit; // max, else prefer ranged query ; TODO or error message ?
      }
      Sort sort = queryParsingContext.getSort();
      if (sort == null) {
         // TODO sort by default : configured in model (last modified date, uri,
         // iri?, types?, owners? other fields...)
         sort = new Sort(Direction.DESC, "_chAt"); // new Sort(Direction.ASC, "_uri")...
      }
      
      Query springMongoQuery = new Query(queryParsingContext.getCriteria())
         .with(sort).skip(start).limit(limit); // TODO rather range query, if possible on sort field
      
      List<DCEntity> foundEntities = executeMongoDbQuery(springMongoQuery, queryParsingContext);
      
      if (logger.isDebugEnabled()) {
         logger.debug("Done Spring Mongo query: " + springMongoQuery
               + "\n   in collection " + dcModel.getCollectionName()
               + " from Model " + dcModel.getName() + " and parameters: " + params
               + "\n   with result nb " + foundEntities.size());
      }
      return foundEntities;
   }

   
   private void parseQueryParameters(Map<String, List<String>> params, DCQueryParsingContext queryParsingContext) {
      DCModel dcModel = queryParsingContext.peekModel();
      
      parameterLoop : for (String fieldPath : params.keySet()) {
         if (findConfParams.contains(fieldPath)) {
            // skip find conf params
            continue;
         }
         
         //DCField dcField = dcModelService.getFieldByPath(dcModel, fieldPath); // TODO TODOOOOOOOOOOOOO
{'fraction_non_alphanumeric': 0.1030787805614247, 'fraction_numerical': 0.005734983398732267, 'mean_word_length': 3.608136300417246, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '686032', 'n_tokens_mistral': 3867, 'n_tokens_neox': 3788, 'n_words': 903}