<commit_before>/*
 * Copyright 2015-2016 OpenCB
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.opencb.opencga.storage.core.local.variant;

import org.apache.commons.lang.StringUtils;
import org.opencb.biodata.models.core.Region;
import org.opencb.biodata.models.variant.Variant;
import org.opencb.biodata.tools.ga4gh.AvroGa4GhVariantFactory;
import org.opencb.biodata.tools.ga4gh.ProtoGa4GhVariantFactory;
import org.opencb.biodata.tools.variant.converter.ga4gh.Ga4ghVariantConverter;
import org.opencb.commons.datastore.core.ObjectMap;
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.commons.datastore.core.QueryResult;
import org.opencb.opencga.catalog.db.api.SampleDBAdaptor;
import org.opencb.opencga.catalog.db.api.StudyDBAdaptor;
import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.managers.CatalogManager;
import org.opencb.opencga.catalog.models.DataStore;
import org.opencb.opencga.catalog.models.File;
import org.opencb.opencga.catalog.models.Sample;
import org.opencb.opencga.catalog.models.Study;
import org.opencb.opencga.storage.core.StorageETLResult;
import org.opencb.opencga.storage.core.config.StorageConfiguration;
import org.opencb.opencga.storage.core.exceptions.StorageManagerException;
import org.opencb.opencga.storage.core.local.StorageManager;
import org.opencb.opencga.storage.core.local.variant.operations.StorageOperation;
import org.opencb.opencga.storage.core.local.variant.operations.VariantAnnotationStorageOperation;
import org.opencb.opencga.storage.core.local.variant.operations.VariantFileIndexerStorageOperation;
import org.opencb.opencga.storage.core.local.variant.operations.VariantStatsStorageOperation;
import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor;
import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptorUtils;
import org.opencb.opencga.storage.core.variant.adaptors.VariantDBIterator;

import java.io.IOException;
import java.net.URISyntaxException;
import java.util.*;
import java.util.function.Function;

public class VariantStorageManager extends StorageManager {

    public static final int LIMIT_DEFAULT = 1000;
    public static final int LIMIT_MAX = 5000;


    public VariantStorageManager(CatalogManager catalogManager, StorageConfiguration storageConfiguration) {
        super(catalogManager, storageConfiguration);

    }

    public void clearCache(String studyId, String type, String sessionId) throws CatalogException {
        String userId = catalogManager.getUserManager().getId(sessionId);

    }

    // -------------------------//
    //   Import/Export methods  //
    // -------------------------//

    public void importData(String fileId, String studyId, String sessionId) {
        throw new UnsupportedOperationException();
    }

    public void exportData(String outputFile, String studyId, String sessionId) {
        throw new UnsupportedOperationException();
    }

    public void exportData(String outputFile, String studyId, Query query, QueryOptions queryOptions, String sessionId) {
        throw new UnsupportedOperationException();
    }

    // --------------------------//
    //   Data Operation methods  //
    // --------------------------//

    public List<StorageETLResult> index(String fileId, String outDir, String catalogOutDir, ObjectMap config, String sessionId)
            throws CatalogException, StorageManagerException, IOException, URISyntaxException {
        return index(Arrays.asList(fileId.split(",")), outDir, catalogOutDir, config, sessionId);
    }

    public List<StorageETLResult> index(List<String> files, String outDir, String catalogOutDir, ObjectMap config, String sessionId)
            throws CatalogException, StorageManagerException, IOException, URISyntaxException {
        VariantFileIndexerStorageOperation indexOperation = new VariantFileIndexerStorageOperation(catalogManager, storageConfiguration);

        List<Long> fileIds = new ArrayList<>(files.size());
        for (String file : files) {
            long fileId = catalogManager.getFileId(file, sessionId);
            fileIds.add(fileId);
        }
        QueryOptions options = new QueryOptions(config);
        options.putIfNotNull(VariantFileIndexerStorageOperation.CATALOG_PATH, catalogOutDir);
        return indexOperation.index(fileIds, outDir, options, sessionId);
    }

    public void deleteStudy(String studyId, String sessionId) {
        throw new UnsupportedOperationException();
    }

    public void deleteFile(String fileId, String studyId, String sessionId) {
        throw new UnsupportedOperationException();
    }

    public void annotate(String study, Query query, String outDir, String catalogOutDir, ObjectMap config, String sessionId)
            throws CatalogException, StorageManagerException, IOException, URISyntaxException {
        VariantAnnotationStorageOperation annotOperation = new VariantAnnotationStorageOperation(catalogManager, storageConfiguration);

        long studyId = catalogManager.getStudyId(study, sessionId);
        annotOperation.annotateVariants(studyId, query, outDir, catalogOutDir, sessionId, config);
    }

    public void deleteAnnotation(String annotationId, String studyId, String sessionId) {
        throw new UnsupportedOperationException();
    }

    public void stats(String study, List<String> cohorts, String outDir, String catalogOutDir, ObjectMap config, String sessionId)
            throws CatalogException, StorageManagerException, IOException, URISyntaxException {
        VariantStatsStorageOperation statsOperation = new VariantStatsStorageOperation(catalogManager, storageConfiguration);

        List<Long> cohortIds = new ArrayList<>(cohorts.size());
        for (String cohort : cohorts) {
            long cohortId = catalogManager.getCohortId(cohort, sessionId);
            cohortIds.add(cohortId);
        }
        long studyId = catalogManager.getStudyId(study, sessionId);
        statsOperation.calculateStats(studyId, cohortIds, outDir, catalogOutDir, new QueryOptions(config), sessionId);
    }

    public void deleteStats(List<String> cohorts, String studyId, String sessionId) {
        throw new UnsupportedOperationException();
    }

    // ---------------------//
    //   Query methods      //
    // ---------------------//

    public QueryResult<Variant> get(Query query, QueryOptions queryOptions, String sessionId)
            throws CatalogException, StorageManagerException, IOException {
        return secure(query, sessionId, dbAdaptor -> {
            addDefaultLimit(queryOptions);
            logger.debug("getVariants {}, {}", query, queryOptions);
            QueryResult<Variant> result = dbAdaptor.get(query, queryOptions);
            logger.debug("gotVariants {}, {}, in {}ms", result.getNumResults(), result.getNumTotalResults(), result.getDbTime());
            return result;<commit_msg>storage: Fix get cohort ids given cohort name<commit_after>/*
 * Copyright 2015-2016 OpenCB
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.opencb.opencga.storage.core.local.variant;

import org.apache.commons.lang.StringUtils;
import org.opencb.biodata.models.core.Region;
import org.opencb.biodata.models.variant.Variant;
import org.opencb.biodata.tools.ga4gh.AvroGa4GhVariantFactory;
import org.opencb.biodata.tools.ga4gh.ProtoGa4GhVariantFactory;
import org.opencb.biodata.tools.variant.converter.ga4gh.Ga4ghVariantConverter;
import org.opencb.commons.datastore.core.ObjectMap;
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.commons.datastore.core.QueryResult;
import org.opencb.opencga.catalog.db.api.SampleDBAdaptor;
import org.opencb.opencga.catalog.db.api.StudyDBAdaptor;
import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.managers.CatalogManager;
import org.opencb.opencga.catalog.models.DataStore;
import org.opencb.opencga.catalog.models.File;
import org.opencb.opencga.catalog.models.Sample;
import org.opencb.opencga.catalog.models.Study;
import org.opencb.opencga.storage.core.StorageETLResult;
import org.opencb.opencga.storage.core.config.StorageConfiguration;
import org.opencb.opencga.storage.core.exceptions.StorageManagerException;
import org.opencb.opencga.storage.core.local.StorageManager;
import org.opencb.opencga.storage.core.local.variant.operations.StorageOperation;
import org.opencb.opencga.storage.core.local.variant.operations.VariantAnnotationStorageOperation;
import org.opencb.opencga.storage.core.local.variant.operations.VariantFileIndexerStorageOperation;
import org.opencb.opencga.storage.core.local.variant.operations.VariantStatsStorageOperation;
import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor;
import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptorUtils;
import org.opencb.opencga.storage.core.variant.adaptors.VariantDBIterator;

import java.io.IOException;
import java.net.URISyntaxException;
import java.util.*;
import java.util.function.Function;

public class VariantStorageManager extends StorageManager {

    public static final int LIMIT_DEFAULT = 1000;
    public static final int LIMIT_MAX = 5000;


    public VariantStorageManager(CatalogManager catalogManager, StorageConfiguration storageConfiguration) {
        super(catalogManager, storageConfiguration);

    }

    public void clearCache(String studyId, String type, String sessionId) throws CatalogException {
        String userId = catalogManager.getUserManager().getId(sessionId);

    }

    // -------------------------//
    //   Import/Export methods  //
    // -------------------------//

    public void importData(String fileId, String studyId, String sessionId) {
        throw new UnsupportedOperationException();
    }

    public void exportData(String outputFile, String studyId, String sessionId) {
        throw new UnsupportedOperationException();
    }

    public void exportData(String outputFile, String studyId, Query query, QueryOptions queryOptions, String sessionId) {
        throw new UnsupportedOperationException();
    }

    // --------------------------//
    //   Data Operation methods  //
    // --------------------------//

    public List<StorageETLResult> index(String fileId, String outDir, String catalogOutDir, ObjectMap config, String sessionId)
            throws CatalogException, StorageManagerException, IOException, URISyntaxException {
        return index(Arrays.asList(fileId.split(",")), outDir, catalogOutDir, config, sessionId);
    }

    public List<StorageETLResult> index(List<String> files, String outDir, String catalogOutDir, ObjectMap config, String sessionId)
            throws CatalogException, StorageManagerException, IOException, URISyntaxException {
        VariantFileIndexerStorageOperation indexOperation = new VariantFileIndexerStorageOperation(catalogManager, storageConfiguration);

        List<Long> fileIds = new ArrayList<>(files.size());
        for (String file : files) {
            long fileId = catalogManager.getFileId(file, sessionId);
            fileIds.add(fileId);
        }
        QueryOptions options = new QueryOptions(config);
        options.putIfNotNull(VariantFileIndexerStorageOperation.CATALOG_PATH, catalogOutDir);
        return indexOperation.index(fileIds, outDir, options, sessionId);
    }

    public void deleteStudy(String studyId, String sessionId) {
        throw new UnsupportedOperationException();
    }

    public void deleteFile(String fileId, String studyId, String sessionId) {
        throw new UnsupportedOperationException();
    }

    public void annotate(String study, Query query, String outDir, String catalogOutDir, ObjectMap config, String sessionId)
            throws CatalogException, StorageManagerException, IOException, URISyntaxException {
        VariantAnnotationStorageOperation annotOperation = new VariantAnnotationStorageOperation(catalogManager, storageConfiguration);

        long studyId = catalogManager.getStudyId(study, sessionId);
        annotOperation.annotateVariants(studyId, query, outDir, catalogOutDir, sessionId, config);
    }

    public void deleteAnnotation(String annotationId, String studyId, String sessionId) {
        throw new UnsupportedOperationException();
    }

    public void stats(String study, List<String> cohorts, String outDir, String catalogOutDir, ObjectMap config, String sessionId)
            throws CatalogException, StorageManagerException, IOException, URISyntaxException {
        VariantStatsStorageOperation statsOperation = new VariantStatsStorageOperation(catalogManager, storageConfiguration);

        List<Long> cohortIds = new ArrayList<>(cohorts.size());
        for (String cohort : cohorts) {
            if (!cohort.contains(":")) {
                cohort = study + ":" + cohort;
            }
            long cohortId = catalogManager.getCohortId(cohort, sessionId);
            cohortIds.add(cohortId);
        }
        long studyId = catalogManager.getStudyId(study, sessionId);
        statsOperation.calculateStats(studyId, cohortIds, outDir, catalogOutDir, new QueryOptions(config), sessionId);
    }

    public void deleteStats(List<String> cohorts, String studyId, String sessionId) {
        throw new UnsupportedOperationException();
    }

    // ---------------------//
    //   Query methods      //
    // ---------------------//

    public QueryResult<Variant> get(Query query, QueryOptions queryOptions, String sessionId)
            throws CatalogException, StorageManagerException, IOException {
        return secure(query, sessionId, dbAdaptor -> {
            addDefaultLimit(queryOptions);
            logger.debug("getVariants {}, {}", query, queryOptions);
            QueryResult<Variant> result = dbAdaptor.get(query, queryOptions);
            logger.debug("gotVariants {}, {}, in {}ms", result.getNumResults(), result.getNumTotalResults(), result.getDbTime());
            return result;
{'fraction_non_alphanumeric': 0.10561812467260345, 'fraction_numerical': 0.0034049240440020955, 'mean_word_length': 3.842422320862397, 'pattern_counts': {'":': 2, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7397892', 'n_tokens_mistral': 4191, 'n_tokens_neox': 4040, 'n_words': 927}