<commit_before>package org.opencb.opencga.app.migrations.v2_4_2.catalog;

import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.model.Filters;
import com.mongodb.client.model.Projections;
import com.mongodb.client.model.Sorts;
import org.apache.commons.collections4.CollectionUtils;
import org.bson.Document;
import org.opencb.commons.datastore.core.ObjectMap;
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor;
import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor;
import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory;
import org.opencb.opencga.catalog.db.mongodb.converters.ClinicalAnalysisConverter;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.migration.Migration;
import org.opencb.opencga.catalog.migration.MigrationTool;
import org.opencb.opencga.core.models.clinical.ClinicalAnalysis;
import org.opencb.opencga.core.models.common.Enums;
import org.opencb.opencga.core.models.individual.Individual;
import org.opencb.opencga.core.models.sample.Sample;

import java.util.*;

@Migration(id = "recover_proband_samples_in_cases_TASK-1470",
        description = "Recover lost samples in clinical collection #TASK-1470", version = "2.4.2",
        language = Migration.MigrationLanguage.JAVA,
        domain = Migration.MigrationDomain.CATALOG,
        date = 20220725)
public class RecoverProbandSamplesInCases extends MigrationTool {

    @Override
    protected void run() throws Exception {
        MongoCollection<Document> audit = getMongoCollection(MongoDBAdaptorFactory.AUDIT_COLLECTION);
        ClinicalAnalysisConverter converter = new ClinicalAnalysisConverter();

        queryMongo(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION,
                new Document("proband.samples", null),
                Projections.include("uid", "id", "uuid", "studyUid", "proband"), (doc) -> {
                    ClinicalAnalysis currentCase = converter.convertToDataModelType(doc);
                    logger.info("Trying to recover Clinical Analysis [id: {}, uid: {}, uuid: {}]", currentCase.getId(),
                            currentCase.getUid(), currentCase.getUuid());

                    if (currentCase.getProband() == null) {
                        logger.warn("Proband not defined in ClinicalAnalysis '{}'. Skipping...", currentCase.getId());
                        return;
                    }

                    ClinicalAnalysis clinicalAnalysis = findUpdatedCaseInAudit(currentCase.getUuid(), currentCase.getProband().getId(),
                            audit, converter);
                    if (clinicalAnalysis == null) {
                        clinicalAnalysis = findCreatedCaseInAudit(currentCase.getUuid(), currentCase.getProband().getId(), audit,
                                converter);
                    }
                    if (clinicalAnalysis == null) {
                        throw new RuntimeException("ClinicalAnalysis '" + currentCase.getId() + "' not found in the audit collection.");
                    }
                    if (clinicalAnalysis.getProband() == null) {
                        throw new RuntimeException("Proband from ClinicalAnalysis '" + currentCase.getId()
                                +  "' could not be found in the audit collection.");
                    }
                    if (CollectionUtils.isEmpty(clinicalAnalysis.getProband().getSamples())) {
                        throw new RuntimeException("List of samples from proband of ClinicalAnalysis '" + currentCase.getId()
                                + "' could not be found in the audit collection.");
                    }

                    // The proband from the audit matches the one used in the case currently :)
                    Individual proband;
                    try {
                        Query query = new Query()
                                .append(IndividualDBAdaptor.QueryParams.UID.key(), currentCase.getProband().getUid())
                                .append(IndividualDBAdaptor.QueryParams.VERSION.key(), currentCase.getProband().getVersion());
                        proband = dbAdaptorFactory.getCatalogIndividualDBAdaptor().get(query,
                                new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(IndividualDBAdaptor.QueryParams.ID.key(),
                                        IndividualDBAdaptor.QueryParams.VERSION.key(), IndividualDBAdaptor.QueryParams.SAMPLES.key())))
                                .first();
                    } catch (CatalogException e) {
                        throw new RuntimeException(e);
                    }
                    List<Sample> sampleList = new ArrayList<>(clinicalAnalysis.getProband().getSamples().size());

                    Map<String, Sample> sampleMap = new HashMap<>();
                    for (Sample sample : proband.getSamples()) {
                        sampleMap.put(sample.getId(), sample);<commit_msg>app: check empty list of samples, #TASK-1751<commit_after>package org.opencb.opencga.app.migrations.v2_4_2.catalog;

import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.model.Filters;
import com.mongodb.client.model.Projections;
import com.mongodb.client.model.Sorts;
import org.apache.commons.collections4.CollectionUtils;
import org.bson.Document;
import org.opencb.commons.datastore.core.ObjectMap;
import org.opencb.commons.datastore.core.Query;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor;
import org.opencb.opencga.catalog.db.api.IndividualDBAdaptor;
import org.opencb.opencga.catalog.db.mongodb.MongoDBAdaptorFactory;
import org.opencb.opencga.catalog.db.mongodb.converters.ClinicalAnalysisConverter;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.migration.Migration;
import org.opencb.opencga.catalog.migration.MigrationTool;
import org.opencb.opencga.core.models.clinical.ClinicalAnalysis;
import org.opencb.opencga.core.models.common.Enums;
import org.opencb.opencga.core.models.individual.Individual;
import org.opencb.opencga.core.models.sample.Sample;

import java.util.*;

@Migration(id = "recover_proband_samples_in_cases_TASK-1470",
        description = "Recover lost samples in clinical collection #TASK-1470", version = "2.4.2",
        language = Migration.MigrationLanguage.JAVA,
        domain = Migration.MigrationDomain.CATALOG,
        date = 20220725,
        patch = 2)
public class RecoverProbandSamplesInCases extends MigrationTool {

    @Override
    protected void run() throws Exception {
        MongoCollection<Document> audit = getMongoCollection(MongoDBAdaptorFactory.AUDIT_COLLECTION);
        ClinicalAnalysisConverter converter = new ClinicalAnalysisConverter();

        queryMongo(MongoDBAdaptorFactory.CLINICAL_ANALYSIS_COLLECTION,
                new Document("$or", Arrays.asList(
                        new Document("proband.samples", null),
                        new Document("proband.samples", Collections.emptyList())
                )),
                Projections.include("uid", "id", "uuid", "studyUid", "proband"), (doc) -> {
                    ClinicalAnalysis currentCase = converter.convertToDataModelType(doc);
                    logger.info("Trying to recover Clinical Analysis [id: {}, uid: {}, uuid: {}]", currentCase.getId(),
                            currentCase.getUid(), currentCase.getUuid());

                    if (currentCase.getProband() == null) {
                        logger.warn("Proband not defined in ClinicalAnalysis '{}'. Skipping...", currentCase.getId());
                        return;
                    }

                    ClinicalAnalysis clinicalAnalysis = findUpdatedCaseInAudit(currentCase.getUuid(), currentCase.getProband().getId(),
                            audit, converter);
                    if (clinicalAnalysis == null) {
                        clinicalAnalysis = findCreatedCaseInAudit(currentCase.getUuid(), currentCase.getProband().getId(), audit,
                                converter);
                    }
                    if (clinicalAnalysis == null) {
                        throw new RuntimeException("ClinicalAnalysis '" + currentCase.getId() + "' not found in the audit collection.");
                    }
                    if (clinicalAnalysis.getProband() == null) {
                        throw new RuntimeException("Proband from ClinicalAnalysis '" + currentCase.getId()
                                +  "' could not be found in the audit collection.");
                    }
                    if (CollectionUtils.isEmpty(clinicalAnalysis.getProband().getSamples())) {
                        throw new RuntimeException("List of samples from proband of ClinicalAnalysis '" + currentCase.getId()
                                + "' could not be found in the audit collection.");
                    }

                    // The proband from the audit matches the one used in the case currently :)
                    Individual proband;
                    try {
                        Query query = new Query()
                                .append(IndividualDBAdaptor.QueryParams.UID.key(), currentCase.getProband().getUid())
                                .append(IndividualDBAdaptor.QueryParams.VERSION.key(), currentCase.getProband().getVersion());
                        proband = dbAdaptorFactory.getCatalogIndividualDBAdaptor().get(query,
                                        new QueryOptions(QueryOptions.INCLUDE, Arrays.asList(IndividualDBAdaptor.QueryParams.ID.key(),
                                                IndividualDBAdaptor.QueryParams.VERSION.key(), IndividualDBAdaptor.QueryParams.SAMPLES.key())))
                                .first();
                    } catch (CatalogException e) {
                        throw new RuntimeException(e);
                    }
                    List<Sample> sampleList = new ArrayList<>(clinicalAnalysis.getProband().getSamples().size());

                    Map<String, Sample> sampleMap = new HashMap<>();
                    for (Sample sample : proband.getSamples()) {
                        sampleMap.put(sample.getId(), sample);
{'fraction_non_alphanumeric': 0.10099369386585133, 'fraction_numerical': 0.004872921842155552, 'mean_word_length': 2.2375502629137025, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2376826', 'n_tokens_mistral': 2846, 'n_tokens_neox': 2663, 'n_words': 481}