<commit_before>package org.opencb.cellbase.mongodb.db;

import com.mongodb.*;
import org.opencb.biodata.models.feature.Region;
import org.opencb.biodata.models.variant.annotation.Clinvar;
import org.opencb.biodata.models.variant.annotation.Cosmic;
import org.opencb.biodata.models.variant.annotation.Gwas;
import org.opencb.biodata.models.variation.GenomicVariant;
import org.opencb.cellbase.core.common.Position;

import org.opencb.cellbase.core.lib.api.variation.ClinicalDBAdaptor;
import org.opencb.datastore.core.QueryOptions;
import org.opencb.datastore.core.QueryResult;
import org.opencb.datastore.mongodb.MongoDataStore;

import java.util.*;
import java.util.regex.Pattern;

/**
 * Created by antonior on 11/18/14.
 * @author <NAME> <EMAIL>
 */
public class ClinicalMongoDBAdaptor extends MongoDBAdaptor implements ClinicalDBAdaptor {


    public ClinicalMongoDBAdaptor(DB db) {
        super(db);
    }

    public ClinicalMongoDBAdaptor(DB db, String species, String assembly) {
        super(db, species, assembly);
        mongoDBCollection = db.getCollection("clinical");
        logger.info("ClinicalVarMongoDBAdaptor: in 'constructor'");
    }

    public ClinicalMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) {
        super(species, assembly, mongoDataStore);
        mongoDBCollection2 = mongoDataStore.getCollection("clinical");

        logger.info("ClinicalMongoDBAdaptor: in 'constructor'");
    }

    @Override
    public QueryResult getAll(QueryOptions options) {
        if(includeContains(options.getAsStringList("include"), "clinvar")) {
            return getAllClinvar(options);
        } else {
            // TODO implement!
            return new QueryResult();
        }
    }

    @Override
    public QueryResult getAllClinvar(QueryOptions options) {
        options.addToListOption("include", "clinvarList");
//        options.addToListOption("include", "chromosome");
//        options.addToListOption("include", "start");
//        options.addToListOption("include", "end");
//        options.addToListOption("include", "reference");
//        options.addToListOption("include", "alternate");
        List<DBObject> pipeline = addClinvarAggregationFilters(options);

//        List<DBObject> pipeline = new ArrayList<>();
//        List<Object> idList = options.getList("id", null);
//        String idString = (String) idList.get(0);
////        pipeline.add(new BasicDBObject("$match", new BasicDBObject("clinvarList.clinvarSet.referenceClinVarAssertion.clinVarAccession.acc", idString)));
//        pipeline.add(new BasicDBObject("$match", new BasicDBObject("clinvarList.clinvarSet.referenceClinVarAssertion.measureSet.measure.measureRelationship.symbol.elementValue.value", "APOE")));
//        pipeline.add(new BasicDBObject("$unwind", "$clinvarList"));
//        pipeline.add(new BasicDBObject("$match", new BasicDBObject("clinvarList.clinvarSet.referenceClinVarAssertion.measureSet.measure.measureRelationship.symbol.elementValue.value", "APOE")));

//        pipeline.add(new BasicDBObject("$match", new BasicDBObject("clinvarList.clinvarSet.referenceClinVarAssertion.clinVarAccession.acc", idString)));

//        addClinvarQueryFilters(builder, options);


        return executeAggregation2("", pipeline, options);
//        return prepareClinvarQueryResultList(Collections.singletonList(executeQuery("result", builder.get(), options))).get(0);
    }

    private List<DBObject> addClinvarAggregationFilters(QueryOptions options) {
        List<DBObject> filterSteps = new ArrayList<>();
        List<DBObject> pipeline = new ArrayList<>();
        filterSteps = addClinvarRcvAggregationFilter(filterSteps, options);
        filterSteps = addClinvarRsAggregationFilter(filterSteps, options);
        filterSteps = addClinvarRegionAggregationFilter(filterSteps, options);
        filterSteps = addClinvarGeneAggregationFilter(filterSteps, options);
        filterSteps = addClinvarPhenotypeAggregationFilter(filterSteps, options);

        // Filtering steps repeated twice to avoid undwind over all clinical records
        pipeline.addAll(filterSteps);
//        pipeline.add(new BasicDBObject("$unwind", "$clinvarList"));
//        pipeline.addAll(filterSteps);
        pipeline.add(new BasicDBObject("$limit", 1000));

        return pipeline;
    }

    private List<DBObject> addClinvarRcvAggregationFilter(List<DBObject> filterSteps, QueryOptions options) {
        List<String> rcvList = (List<String>) options.get("rcv");
        if (rcvList != null && rcvList.size() > 0) {
            logger.info("rcv filter activated, rcv list: "+rcvList.toString());
            filterSteps.add(new BasicDBObject("$match",
//                    new BasicDBObject("clinvarList.clinvarSet.referenceClinVarAssertion.clinVarAccession.acc",
                    new BasicDBObject("clinvarSet.referenceClinVarAssertion.clinVarAccession.acc",
                            new BasicDBObject("$in", rcvList))));
        }
        return filterSteps;
    }

    private List<DBObject> addClinvarRsAggregationFilter(List<DBObject> filterSteps, QueryOptions options) {
//        List<String> rsStringList = options.getAsStringList("rs", null);
        List<String> rsStringList = (List<String>) options.get("rs");
        if (rsStringList != null && rsStringList.size() > 0) {
            logger.info("rs filter activated, res list: "+rsStringList.toString());
            List<String> rsList = new ArrayList<>(rsStringList.size());<commit_msg>feature/clinvarReorganization: projection over the clinvarSet attribute added to getAllClinvar<commit_after>package org.opencb.cellbase.mongodb.db;

import com.mongodb.*;
import org.opencb.biodata.models.feature.Region;
import org.opencb.biodata.models.variant.annotation.Clinvar;
import org.opencb.biodata.models.variant.annotation.Cosmic;
import org.opencb.biodata.models.variant.annotation.Gwas;
import org.opencb.biodata.models.variation.GenomicVariant;
import org.opencb.cellbase.core.common.Position;

import org.opencb.cellbase.core.lib.api.variation.ClinicalDBAdaptor;
import org.opencb.datastore.core.QueryOptions;
import org.opencb.datastore.core.QueryResult;
import org.opencb.datastore.mongodb.MongoDataStore;

import java.util.*;
import java.util.regex.Pattern;

/**
 * Created by antonior on 11/18/14.
 * @author <NAME> <EMAIL>
 */
public class ClinicalMongoDBAdaptor extends MongoDBAdaptor implements ClinicalDBAdaptor {


    public ClinicalMongoDBAdaptor(DB db) {
        super(db);
    }

    public ClinicalMongoDBAdaptor(DB db, String species, String assembly) {
        super(db, species, assembly);
        mongoDBCollection = db.getCollection("clinical");
        logger.info("ClinicalVarMongoDBAdaptor: in 'constructor'");
    }

    public ClinicalMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) {
        super(species, assembly, mongoDataStore);
        mongoDBCollection2 = mongoDataStore.getCollection("clinical");

        logger.info("ClinicalMongoDBAdaptor: in 'constructor'");
    }

    @Override
    public QueryResult getAll(QueryOptions options) {
        if(includeContains(options.getAsStringList("include"), "clinvar")) {
            return getAllClinvar(options);
        } else {
            // TODO implement!
            return new QueryResult();
        }
    }

    @Override
    public QueryResult getAllClinvar(QueryOptions options) {
//        options.addToListOption("include", "clinvarList");
//        options.addToListOption("include", "chromosome");
//        options.addToListOption("include", "start");
//        options.addToListOption("include", "end");
//        options.addToListOption("include", "reference");
//        options.addToListOption("include", "alternate");
        List<DBObject> pipeline = new ArrayList<>();
        pipeline = addClinvarAggregationFilters(pipeline, options);
        DBObject fields = new BasicDBObject();
        fields.put("clinvarSet", 1);
        fields.put("chromosome", 1);
        fields.put("start", 1);
        fields.put("end", 1);
        fields.put("reference", 1);
        fields.put("alternate", 1);
        pipeline.add(new BasicDBObject("$project", fields));


//        List<DBObject> pipeline = new ArrayList<>();
//        List<Object> idList = options.getList("id", null);
//        String idString = (String) idList.get(0);
////        pipeline.add(new BasicDBObject("$match", new BasicDBObject("clinvarList.clinvarSet.referenceClinVarAssertion.clinVarAccession.acc", idString)));
//        pipeline.add(new BasicDBObject("$match", new BasicDBObject("clinvarList.clinvarSet.referenceClinVarAssertion.measureSet.measure.measureRelationship.symbol.elementValue.value", "APOE")));
//        pipeline.add(new BasicDBObject("$unwind", "$clinvarList"));
//        pipeline.add(new BasicDBObject("$match", new BasicDBObject("clinvarList.clinvarSet.referenceClinVarAssertion.measureSet.measure.measureRelationship.symbol.elementValue.value", "APOE")));

//        pipeline.add(new BasicDBObject("$match", new BasicDBObject("clinvarList.clinvarSet.referenceClinVarAssertion.clinVarAccession.acc", idString)));

//        addClinvarQueryFilters(builder, options);


        return executeAggregation2("", pipeline, options);
//        return prepareClinvarQueryResultList(Collections.singletonList(executeQuery("result", builder.get(), options))).get(0);
    }

    private List<DBObject> addClinvarAggregationFilters(List<DBObject> pipeline, QueryOptions options) {
        List<DBObject> filterSteps = new ArrayList<>();
        filterSteps.add(new BasicDBObject("$match", new BasicDBObject("clinvarSet", new BasicDBObject("$exists", 1))));
        filterSteps = addClinvarRcvAggregationFilter(filterSteps, options);
        filterSteps = addClinvarRsAggregationFilter(filterSteps, options);
        filterSteps = addClinvarRegionAggregationFilter(filterSteps, options);
        filterSteps = addClinvarGeneAggregationFilter(filterSteps, options);
        filterSteps = addClinvarPhenotypeAggregationFilter(filterSteps, options);

        // Filtering steps repeated twice to avoid undwind over all clinical records
        pipeline.addAll(filterSteps);
//        pipeline.add(new BasicDBObject("$unwind", "$clinvarList"));
//        pipeline.addAll(filterSteps);
        pipeline.add(new BasicDBObject("$limit", 100));

        return pipeline;
    }

    private List<DBObject> addClinvarRcvAggregationFilter(List<DBObject> filterSteps, QueryOptions options) {
        List<String> rcvList = (List<String>) options.get("rcv");
        if (rcvList != null && rcvList.size() > 0) {
            logger.info("rcv filter activated, rcv list: "+rcvList.toString());
            filterSteps.add(new BasicDBObject("$match",
//                    new BasicDBObject("clinvarList.clinvarSet.referenceClinVarAssertion.clinVarAccession.acc",
                    new BasicDBObject("clinvarSet.referenceClinVarAssertion.clinVarAccession.acc",
                            new BasicDBObject("$in", rcvList))));
        }
        return filterSteps;
    }

    private List<DBObject> addClinvarRsAggregationFilter(List<DBObject> filterSteps, QueryOptions options) {
//        List<String> rsStringList = options.getAsStringList("rs", null);
        List<String> rsStringList = (List<String>) options.get("rs");
        if (rsStringList != null && rsStringList.size() > 0) {
            logger.info("rs filter activated, res list: "+rsStringList.toString());
            List<String> rsList = new ArrayList<>(rsStringList.size());
{'fraction_non_alphanumeric': 0.12379963664676875, 'fraction_numerical': 0.003287481616056752, 'mean_word_length': 3.8776371308016877, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1923416', 'n_tokens_mistral': 3478, 'n_tokens_neox': 3365, 'n_words': 567}