<commit_before>package com.deutscheboerse.risk.dave.persistence;

import CIL.CIL_v001.Prisma_v001.PrismaReports;
import CIL.ObjectList;
import com.deutscheboerse.risk.dave.BaseTest;
import com.deutscheboerse.risk.dave.MainVerticleIT;
import com.deutscheboerse.risk.dave.model.*;
import com.google.protobuf.ExtensionRegistry;
import io.vertx.core.Vertx;
import io.vertx.core.eventbus.MessageConsumer;
import io.vertx.core.json.JsonObject;
import io.vertx.ext.mongo.FindOptions;
import io.vertx.ext.mongo.MongoClient;
import io.vertx.ext.unit.Async;
import io.vertx.ext.unit.TestContext;
import io.vertx.ext.unit.junit.VertxUnitRunner;
import io.vertx.serviceproxy.ProxyHelper;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.function.BiConsumer;

@RunWith(VertxUnitRunner.class)
public class MongoPersistenceServiceIT extends BaseTest {
    private static Vertx vertx;
    private static MongoClient mongoClient;
    private static PersistenceService persistenceProxy;
    private static final double DOUBLE_DELTA = 1e-12;
    private static MessageConsumer<JsonObject> persistenceServiceConsumer;

    @BeforeClass
    public static void setUp(TestContext context) {
        MongoPersistenceServiceIT.vertx = Vertx.vertx();
        JsonObject config = BaseTest.getMongoConfig();
        JsonObject mongoConfig = BaseTest.getMongoClientConfig(config);

        MongoPersistenceServiceIT.mongoClient = MongoClient.createShared(MongoPersistenceServiceIT.vertx, mongoConfig);

        PersistenceService mongoPersistenceService = new MongoPersistenceService(vertx, config);

        MongoPersistenceServiceIT.persistenceServiceConsumer = ProxyHelper.registerService(PersistenceService.class, vertx, mongoPersistenceService, PersistenceService.SERVICE_ADDRESS);
        MongoPersistenceServiceIT.persistenceProxy = ProxyHelper.createProxy(PersistenceService.class, vertx, PersistenceService.SERVICE_ADDRESS);

        mongoPersistenceService.initialize(context.asyncAssertSuccess());
    }

    @Test
    public void checkCollectionsExist(TestContext context) {
        List<String> requiredCollections = new ArrayList<>();
        requiredCollections.add(AccountMarginModel.MONGO_HISTORY_COLLECTION);
        requiredCollections.add(AccountMarginModel.MONGO_LATEST_COLLECTION);
        requiredCollections.add(LiquiGroupMarginModel.MONGO_HISTORY_COLLECTION);
        requiredCollections.add(LiquiGroupMarginModel.MONGO_LATEST_COLLECTION);
        requiredCollections.add(LiquiGroupSplitMarginModel.MONGO_HISTORY_COLLECTION);
        requiredCollections.add(LiquiGroupSplitMarginModel.MONGO_LATEST_COLLECTION);
        requiredCollections.add(PoolMarginModel.MONGO_HISTORY_COLLECTION);
        requiredCollections.add(PoolMarginModel.MONGO_LATEST_COLLECTION);
        requiredCollections.add(RiskLimitUtilizationModel.MONGO_HISTORY_COLLECTION);
        requiredCollections.add(RiskLimitUtilizationModel.MONGO_LATEST_COLLECTION);
        final Async async = context.async();
        MongoPersistenceServiceIT.mongoClient.getCollections(ar -> {
            if (ar.succeeded()) {
                if (ar.result().containsAll(requiredCollections)) {
                    async.complete();
                } else {
                    requiredCollections.removeAll(ar.result());
                    context.fail("Following collections were not created: " + requiredCollections);
                }
            } else {
                context.fail(ar.cause());
            }
        });
    }

    @Test
    public void testAccountMarginStore(TestContext context) throws IOException {
        Async asyncStore1 = context.async(1704);
        readTTSaveFile("accountMargin", 1, (header, gpbObject) -> {
            PrismaReports.AccountMargin accountMarginData = gpbObject.getExtension(PrismaReports.accountMargin);
            AccountMarginModel accountMarginModel = new AccountMarginModel(header, accountMarginData);
            persistenceProxy.store(accountMarginModel, ModelType.ACCOUNT_MARGIN_MODEL, ar -> {
                if (ar.succeeded()) {
                    asyncStore1.countDown();
                } else {
                    context.fail(ar.cause());
                }
            });
        });
        asyncStore1.awaitSuccess(30000);

        Async asyncStore2 = context.async(1704);
        readTTSaveFile("accountMargin", 2, (header, gpbObject) -> {
            PrismaReports.AccountMargin accountMarginData = gpbObject.getExtension(PrismaReports.accountMargin);
            AccountMarginModel accountMarginModel = new AccountMarginModel(header, accountMarginData);
            persistenceProxy.store(accountMarginModel, ModelType.ACCOUNT_MARGIN_MODEL, ar -> {
                if (ar.succeeded()) {
                    asyncStore2.countDown();
                } else {
                    context.fail(ar.cause());<commit_msg>Tests for existence of indexes #20<commit_after>package com.deutscheboerse.risk.dave.persistence;

import CIL.CIL_v001.Prisma_v001.PrismaReports;
import CIL.ObjectList;
import com.deutscheboerse.risk.dave.BaseTest;
import com.deutscheboerse.risk.dave.MainVerticleIT;
import com.deutscheboerse.risk.dave.model.*;
import com.google.protobuf.ExtensionRegistry;
import io.vertx.core.Vertx;
import io.vertx.core.eventbus.MessageConsumer;
import io.vertx.core.json.JsonArray;
import io.vertx.core.json.JsonObject;
import io.vertx.ext.mongo.FindOptions;
import io.vertx.ext.mongo.MongoClient;
import io.vertx.ext.unit.Async;
import io.vertx.ext.unit.TestContext;
import io.vertx.ext.unit.junit.VertxUnitRunner;
import io.vertx.serviceproxy.ProxyHelper;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.function.BiConsumer;

@RunWith(VertxUnitRunner.class)
public class MongoPersistenceServiceIT extends BaseTest {
    private static Vertx vertx;
    private static MongoClient mongoClient;
    private static PersistenceService persistenceProxy;
    private static final double DOUBLE_DELTA = 1e-12;
    private static MessageConsumer<JsonObject> persistenceServiceConsumer;

    @BeforeClass
    public static void setUp(TestContext context) {
        MongoPersistenceServiceIT.vertx = Vertx.vertx();
        JsonObject config = BaseTest.getMongoConfig();
        JsonObject mongoConfig = BaseTest.getMongoClientConfig(config);

        MongoPersistenceServiceIT.mongoClient = MongoClient.createShared(MongoPersistenceServiceIT.vertx, mongoConfig);

        PersistenceService mongoPersistenceService = new MongoPersistenceService(vertx, config);

        MongoPersistenceServiceIT.persistenceServiceConsumer = ProxyHelper.registerService(PersistenceService.class, vertx, mongoPersistenceService, PersistenceService.SERVICE_ADDRESS);
        MongoPersistenceServiceIT.persistenceProxy = ProxyHelper.createProxy(PersistenceService.class, vertx, PersistenceService.SERVICE_ADDRESS);

        mongoPersistenceService.initialize(context.asyncAssertSuccess());
    }

    @Test
    public void checkCollectionsExist(TestContext context) {
        List<String> requiredCollections = new ArrayList<>();
        requiredCollections.add(AccountMarginModel.MONGO_HISTORY_COLLECTION);
        requiredCollections.add(AccountMarginModel.MONGO_LATEST_COLLECTION);
        requiredCollections.add(LiquiGroupMarginModel.MONGO_HISTORY_COLLECTION);
        requiredCollections.add(LiquiGroupMarginModel.MONGO_LATEST_COLLECTION);
        requiredCollections.add(LiquiGroupSplitMarginModel.MONGO_HISTORY_COLLECTION);
        requiredCollections.add(LiquiGroupSplitMarginModel.MONGO_LATEST_COLLECTION);
        requiredCollections.add(PoolMarginModel.MONGO_HISTORY_COLLECTION);
        requiredCollections.add(PoolMarginModel.MONGO_LATEST_COLLECTION);
        requiredCollections.add(RiskLimitUtilizationModel.MONGO_HISTORY_COLLECTION);
        requiredCollections.add(RiskLimitUtilizationModel.MONGO_LATEST_COLLECTION);
        final Async async = context.async();
        MongoPersistenceServiceIT.mongoClient.getCollections(ar -> {
            if (ar.succeeded()) {
                if (ar.result().containsAll(requiredCollections)) {
                    async.complete();
                } else {
                    requiredCollections.removeAll(ar.result());
                    context.fail("Following collections were not created: " + requiredCollections);
                }
            } else {
                context.fail(ar.cause());
            }
        });
    }

    @Test
    public void checkIndexesExist(TestContext context) {
        List<MongoModel> requiredModels = new ArrayList<>();
        requiredModels.add(new AccountMarginModel());
        requiredModels.add(new LiquiGroupMarginModel());
        requiredModels.add(new LiquiGroupSplitMarginModel());
        requiredModels.add(new PoolMarginModel());
        requiredModels.add(new PositionReportModel());
        requiredModels.add(new RiskLimitUtilizationModel());
        // one index for history and one for latest collection in each model
        final Async async = context.async(requiredModels.size() * 2);
        BiConsumer<String, JsonObject> indexCheck = (collectionName, expectedIndex) -> {
            MongoPersistenceServiceIT.mongoClient.listIndexes(collectionName, ar -> {
                if (ar.succeeded()) {
                    JsonArray result = ar.result();
                    Optional<Object> latestUniqueIndex = result.stream()
                            .filter(index -> index instanceof JsonObject)
                            .filter(index -> ((JsonObject) index).getString("name", "").equals("unique_idx"))
                            .filter(index -> ((JsonObject) index).getJsonObject("key", new JsonObject()).equals(expectedIndex))
                            .findFirst();
                    if (latestUniqueIndex.isPresent()) {
                        async.countDown();
                    } else {
                        context.fail("Missing unique index for collection " + collectionName);
                    }
                } else {
                    context.fail("Unable to list indexes from collection " + collectionName);
                }
            });
        };

        requiredModels.forEach(model -> {
            indexCheck.accept(model.getHistoryCollection(), model.getHistoryUniqueIndex());
            indexCheck.accept(model.getLatestCollection(), model.getLatestUniqueIndex());
        });
    }

    @Test
    public void testAccountMarginStore(TestContext context) throws IOException {
        Async asyncStore1 = context.async(1704);
        readTTSaveFile("accountMargin", 1, (header, gpbObject) -> {
            PrismaReports.AccountMargin accountMarginData = gpbObject.getExtension(PrismaReports.accountMargin);
            AccountMarginModel accountMarginModel = new AccountMarginModel(header, accountMarginData);
            persistenceProxy.store(accountMarginModel, ModelType.ACCOUNT_MARGIN_MODEL, ar -> {
                if (ar.succeeded()) {
                    asyncStore1.countDown();
                } else {
                    context.fail(ar.cause());
                }
            });
        });
        asyncStore1.awaitSuccess(30000);

        Async asyncStore2 = context.async(1704);
        readTTSaveFile("accountMargin", 2, (header, gpbObject) -> {
            PrismaReports.AccountMargin accountMarginData = gpbObject.getExtension(PrismaReports.accountMargin);
            AccountMarginModel accountMarginModel = new AccountMarginModel(header, accountMarginData);
            persistenceProxy.store(accountMarginModel, ModelType.ACCOUNT_MARGIN_MODEL, ar -> {
                if (ar.succeeded()) {
                    asyncStore2.countDown();
                } else {
                    context.fail(ar.cause());
{'fraction_non_alphanumeric': 0.09007897334649556, 'fraction_numerical': 0.0050180980585718985, 'mean_word_length': 3.210945618288881, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6350124', 'n_tokens_mistral': 3384, 'n_tokens_neox': 3287, 'n_words': 508}