<commit_before>package us.kbase.workspace.test.database.mongo;

import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.File;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.jongo.Jongo;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;

import us.kbase.common.mongo.GetMongoDB;
import us.kbase.common.service.UObject;
import us.kbase.common.test.controllers.mongo.MongoController;
import us.kbase.typedobj.core.AbsoluteTypeDefId;
import us.kbase.typedobj.core.LocalTypeProvider;
import us.kbase.typedobj.core.TempFilesManager;
import us.kbase.typedobj.core.TypeDefId;
import us.kbase.typedobj.core.TypeDefName;
import us.kbase.typedobj.core.TypedObjectValidator;
import us.kbase.typedobj.db.MongoTypeStorage;
import us.kbase.typedobj.db.TypeDefinitionDB;
import us.kbase.typedobj.idref.IdReferenceHandlerSetFactory;
import us.kbase.typedobj.idref.IdReferenceType;
import us.kbase.typedobj.idref.RemappedId;
import us.kbase.typedobj.test.DummyTypedObjectValidationReport;
import us.kbase.workspace.database.DefaultReferenceParser;
import us.kbase.workspace.database.ObjectIDNoWSNoVer;
import us.kbase.workspace.database.ObjectIDResolvedWS;
import us.kbase.workspace.database.ObjectIdentifier;
import us.kbase.workspace.database.ObjectInformation;
import us.kbase.workspace.database.Provenance;
import us.kbase.workspace.database.Reference;
import us.kbase.workspace.database.ResolvedSaveObject;
import us.kbase.workspace.database.ResourceUsageConfigurationBuilder;
import us.kbase.workspace.database.Types;
import us.kbase.workspace.database.Workspace;
import us.kbase.workspace.database.WorkspaceIdentifier;
import us.kbase.workspace.database.WorkspaceSaveObject;
import us.kbase.workspace.database.WorkspaceUser;
import us.kbase.workspace.database.WorkspaceUserMetadata;
import us.kbase.workspace.database.exceptions.CorruptWorkspaceDBException;
import us.kbase.workspace.database.exceptions.NoSuchObjectException;
import us.kbase.workspace.database.exceptions.WorkspaceDBInitializationException;
import us.kbase.workspace.database.mongo.GridFSBlobStore;
import us.kbase.workspace.database.mongo.IDName;
import us.kbase.workspace.database.mongo.MongoWorkspaceDB;
import us.kbase.workspace.database.mongo.ObjectSavePackage;
import us.kbase.workspace.database.mongo.ResolvedMongoWSID;
import us.kbase.workspace.database.mongo.TypeData;
import us.kbase.workspace.test.WorkspaceTestCommon;

import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.MongoClient;

public class MongoInternalsTest {
	
	private static Jongo jdb;
	private static MongoWorkspaceDB mwdb;
	private static Workspace ws;
	private static Types types;
	private static MongoController mongo;
	private static MongoClient mongoClient;
	
	private static final IdReferenceHandlerSetFactory fac =
			new IdReferenceHandlerSetFactory(100);
	
	public static final TypeDefId SAFE_TYPE =
			new TypeDefId(new TypeDefName("SomeModule", "AType"), 0, 1);

	@BeforeClass
	public static void setUpClass() throws Exception {
		mongo = new MongoController(WorkspaceTestCommon.getMongoExe(),
				Paths.get(WorkspaceTestCommon.getTempDir()),
				WorkspaceTestCommon.useWiredTigerEngine());
		System.out.println("Using mongo temp dir " +
				mongo.getTempDir());
		WorkspaceTestCommon.stfuLoggers();
		String mongohost = "localhost:" + mongo.getServerPort();
		mongoClient = new MongoClient(mongohost);
		final DB db = mongoClient.getDB("MongoInternalsTest");
		String typedb = "MongoInternalsTest_types";
		WorkspaceTestCommon.destroyWSandTypeDBs(db, typedb);
		jdb = new Jongo(db);
		
		TempFilesManager tfm = new TempFilesManager(
				new File(WorkspaceTestCommon.getTempDir()));
		final TypeDefinitionDB typeDefDB = new TypeDefinitionDB(
				new MongoTypeStorage(GetMongoDB.getDB(mongohost, typedb)));
		TypedObjectValidator val = new TypedObjectValidator(
				new LocalTypeProvider(typeDefDB));
		mwdb = new MongoWorkspaceDB(db, new GridFSBlobStore(db), tfm);
		ws = new Workspace(mwdb,
				new ResourceUsageConfigurationBuilder().build(),
				new DefaultReferenceParser(), val);
		assertTrue("GridFS backend setup failed",
				ws.getBackendType().equals("GridFS"));

		//make a general spec that tests that don't worry about typechecking can use
		WorkspaceUser foo = new WorkspaceUser("foo");
		//simple spec
		types = new Types(typeDefDB);
		types.requestModuleRegistration(foo, "SomeModule");
		types.resolveModuleRegistration("SomeModule", true);
		types.compileNewTypeSpec(foo, 
				"module SomeModule {/* @optional thing */ typedef structure {string thing;} AType;};",
				Arrays.asList("AType"), null, null, false, null);
		types.releaseTypes(foo, "SomeModule");
	}
	
	@AfterClass
	public static void tearDownClass() throws Exception {
		if (mongo != null) {
			mongo.destroy(WorkspaceTestCommon.deleteTempFiles());
		}
	}
	
	@Test
	public void startUpAndCheckConfigDoc() throws Exception {
		final DB db = mongoClient.getDB("startUpAndCheckConfigDoc");
		TempFilesManager tfm = new TempFilesManager(
				new File(WorkspaceTestCommon.getTempDir()));
		new MongoWorkspaceDB(db, new GridFSBlobStore(db), tfm);
		
		DBCursor c = db.getCollection("config").find();
		assertThat("Only one config doc", c.size(), is(1));
		DBObject cd = c.next();
		assertThat("correct config key & value", (String)cd.get("config"),
				is("config"));
		assertThat("not in update", (Boolean)cd.get("inupdate"), is(false));
		assertThat("schema v1", (Integer)cd.get("schemaver"), is(1));
	}
	
	@Test
	public void startUpWith2ConfigDocs() throws Exception {
		final DB db = mongoClient.getDB("startUpWith2ConfigDocs");
		
		final Map<String, Object> m = new HashMap<String, Object>();
		m.put("config", "config");
		m.put("inupdate", false);
		m.put("schemaver", 1);
		
		db.getCollection("config").insert(Arrays.asList(
				(DBObject) new BasicDBObject(m), new BasicDBObject(m)));
		
		failMongoWSStart(db, new CorruptWorkspaceDBException(
				"Found duplicate index keys in the database, " +
						"aborting startup"));
	}
	
	@Test
	public void startUpWithBadSchemaVersion() throws Exception {
		final DB db = mongoClient.getDB("startUpWithBadSchemaVersion");
		<commit_msg>Check startup with existing correct config obj<commit_after>package us.kbase.workspace.test.database.mongo;

import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.File;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.jongo.Jongo;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;

import us.kbase.common.mongo.GetMongoDB;
import us.kbase.common.service.UObject;
import us.kbase.common.test.controllers.mongo.MongoController;
import us.kbase.typedobj.core.AbsoluteTypeDefId;
import us.kbase.typedobj.core.LocalTypeProvider;
import us.kbase.typedobj.core.TempFilesManager;
import us.kbase.typedobj.core.TypeDefId;
import us.kbase.typedobj.core.TypeDefName;
import us.kbase.typedobj.core.TypedObjectValidator;
import us.kbase.typedobj.db.MongoTypeStorage;
import us.kbase.typedobj.db.TypeDefinitionDB;
import us.kbase.typedobj.idref.IdReferenceHandlerSetFactory;
import us.kbase.typedobj.idref.IdReferenceType;
import us.kbase.typedobj.idref.RemappedId;
import us.kbase.typedobj.test.DummyTypedObjectValidationReport;
import us.kbase.workspace.database.DefaultReferenceParser;
import us.kbase.workspace.database.ObjectIDNoWSNoVer;
import us.kbase.workspace.database.ObjectIDResolvedWS;
import us.kbase.workspace.database.ObjectIdentifier;
import us.kbase.workspace.database.ObjectInformation;
import us.kbase.workspace.database.Provenance;
import us.kbase.workspace.database.Reference;
import us.kbase.workspace.database.ResolvedSaveObject;
import us.kbase.workspace.database.ResourceUsageConfigurationBuilder;
import us.kbase.workspace.database.Types;
import us.kbase.workspace.database.Workspace;
import us.kbase.workspace.database.WorkspaceIdentifier;
import us.kbase.workspace.database.WorkspaceInformation;
import us.kbase.workspace.database.WorkspaceSaveObject;
import us.kbase.workspace.database.WorkspaceUser;
import us.kbase.workspace.database.WorkspaceUserMetadata;
import us.kbase.workspace.database.exceptions.CorruptWorkspaceDBException;
import us.kbase.workspace.database.exceptions.NoSuchObjectException;
import us.kbase.workspace.database.exceptions.WorkspaceDBInitializationException;
import us.kbase.workspace.database.mongo.GridFSBlobStore;
import us.kbase.workspace.database.mongo.IDName;
import us.kbase.workspace.database.mongo.MongoWorkspaceDB;
import us.kbase.workspace.database.mongo.ObjectSavePackage;
import us.kbase.workspace.database.mongo.ResolvedMongoWSID;
import us.kbase.workspace.database.mongo.TypeData;
import us.kbase.workspace.test.WorkspaceTestCommon;

import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.MongoClient;

public class MongoInternalsTest {
	
	private static Jongo jdb;
	private static MongoWorkspaceDB mwdb;
	private static Workspace ws;
	private static Types types;
	private static MongoController mongo;
	private static MongoClient mongoClient;
	
	private static final IdReferenceHandlerSetFactory fac =
			new IdReferenceHandlerSetFactory(100);
	
	public static final TypeDefId SAFE_TYPE =
			new TypeDefId(new TypeDefName("SomeModule", "AType"), 0, 1);

	@BeforeClass
	public static void setUpClass() throws Exception {
		mongo = new MongoController(WorkspaceTestCommon.getMongoExe(),
				Paths.get(WorkspaceTestCommon.getTempDir()),
				WorkspaceTestCommon.useWiredTigerEngine());
		System.out.println("Using mongo temp dir " +
				mongo.getTempDir());
		WorkspaceTestCommon.stfuLoggers();
		String mongohost = "localhost:" + mongo.getServerPort();
		mongoClient = new MongoClient(mongohost);
		final DB db = mongoClient.getDB("MongoInternalsTest");
		String typedb = "MongoInternalsTest_types";
		WorkspaceTestCommon.destroyWSandTypeDBs(db, typedb);
		jdb = new Jongo(db);
		
		TempFilesManager tfm = new TempFilesManager(
				new File(WorkspaceTestCommon.getTempDir()));
		final TypeDefinitionDB typeDefDB = new TypeDefinitionDB(
				new MongoTypeStorage(GetMongoDB.getDB(mongohost, typedb)));
		TypedObjectValidator val = new TypedObjectValidator(
				new LocalTypeProvider(typeDefDB));
		mwdb = new MongoWorkspaceDB(db, new GridFSBlobStore(db), tfm);
		ws = new Workspace(mwdb,
				new ResourceUsageConfigurationBuilder().build(),
				new DefaultReferenceParser(), val);
		assertTrue("GridFS backend setup failed",
				ws.getBackendType().equals("GridFS"));

		//make a general spec that tests that don't worry about typechecking can use
		WorkspaceUser foo = new WorkspaceUser("foo");
		//simple spec
		types = new Types(typeDefDB);
		types.requestModuleRegistration(foo, "SomeModule");
		types.resolveModuleRegistration("SomeModule", true);
		types.compileNewTypeSpec(foo, 
				"module SomeModule {/* @optional thing */ typedef structure {string thing;} AType;};",
				Arrays.asList("AType"), null, null, false, null);
		types.releaseTypes(foo, "SomeModule");
	}
	
	@AfterClass
	public static void tearDownClass() throws Exception {
		if (mongo != null) {
			mongo.destroy(WorkspaceTestCommon.deleteTempFiles());
		}
	}
	
	@Test
	public void startUpAndCheckConfigDoc() throws Exception {
		final DB db = mongoClient.getDB("startUpAndCheckConfigDoc");
		TempFilesManager tfm = new TempFilesManager(
				new File(WorkspaceTestCommon.getTempDir()));
		new MongoWorkspaceDB(db, new GridFSBlobStore(db), tfm);
		
		DBCursor c = db.getCollection("config").find();
		assertThat("Only one config doc", c.size(), is(1));
		DBObject cd = c.next();
		assertThat("correct config key & value", (String)cd.get("config"),
				is("config"));
		assertThat("not in update", (Boolean)cd.get("inupdate"), is(false));
		assertThat("schema v1", (Integer)cd.get("schemaver"), is(1));
		
		//check startup works with the config object in place
		MongoWorkspaceDB m = new MongoWorkspaceDB(
				db,  new GridFSBlobStore(db), tfm);
		WorkspaceInformation ws = m.createWorkspace(
				new WorkspaceUser("foo"), "bar", false, null,
				new WorkspaceUserMetadata());
		assertThat("check a ws field", ws.getName(), is("bar"));
		
	}
	
	@Test
	public void startUpWith2ConfigDocs() throws Exception {
		final DB db = mongoClient.getDB("startUpWith2ConfigDocs");
		
		final Map<String, Object> m = new HashMap<String, Object>();
		m.put("config", "config");
		m.put("inupdate", false);
		m.put("schemaver", 1);
		
		db.getCollection("config").insert(Arrays.asList(
				(DBObject) new BasicDBObject(m), new BasicDBObject(m)));
		
		failMongoWSStart(db, new CorruptWorkspaceDBException(
				"Found duplicate index keys in the database, " +
						"aborting startup"));
	}
	
	@Test
	public void startUpWithBadSchemaVersion() throws Exception {
		final DB db = mongoClient.getDB("startUpWithBadSchemaVersion");
		
{'fraction_non_alphanumeric': 0.11067136792113406, 'fraction_numerical': 0.0015830754839173922, 'mean_word_length': 5.890431333663857, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6272423', 'n_tokens_mistral': 4607, 'n_tokens_neox': 4344, 'n_words': 875}