<commit_before>package controllers.api;

import gov.nrel.util.SearchUtils;
import gov.nrel.util.Utility;

import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;

import javax.xml.parsers.ParserConfigurationException;

import models.Entity;
import models.EntityGroup;
import models.EntityGroupXref;
import models.EntityUser;
import models.KeyToTableName;
import models.PermissionType;
import models.RoleMapping;
import models.SdiColumn;
import models.SdiColumnProperty;
import models.SecureResource;
import models.SecureResourceGroupXref;
import models.SecureSchema;
import models.SecureTable;
import models.SecurityGroup;
import models.message.DatasetColumnModel;
import models.message.DatasetColumnSemanticModel;
import models.message.DatasetType;
import models.message.RegisterMessage;
import models.message.RegisterResponseMessage;
import models.message.SearchableParent;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.CoreAdminResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
import org.joda.time.LocalDateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;

import play.Play;
import play.mvc.Util;
import play.mvc.results.BadRequest;

import com.alvazan.orm.api.base.NoSqlEntityManager;
import com.alvazan.orm.api.z8spi.meta.DboColumnCommonMeta;
import com.alvazan.orm.api.z8spi.meta.DboColumnIdMeta;
import com.alvazan.orm.api.z8spi.meta.DboColumnMeta;
import com.alvazan.orm.api.z8spi.meta.DboColumnToOneMeta;
import com.alvazan.orm.api.z8spi.meta.DboTableMeta;
import com.alvazan.orm.api.z8spi.meta.TypedColumn;
import com.alvazan.play.NoSql;

import controllers.SearchPosting;
import controllers.SecurityUtil;

public class ApiRegistrationImpl {

	private static final Logger log = LoggerFactory.getLogger(ApiRegistrationImpl.class);
	public static int divisor = Integer.MAX_VALUE / 10;

	public static RegisterResponseMessage registerImpl(RegisterMessage msg, String username, String apiKey) {
		String mode = (String) Play.configuration.get("upgrade.mode");
		String requestUrl = null;
		if(mode == null) {
		} if(mode.startsWith("http")) {
			requestUrl = mode;
		} else if("NEW".equals(mode)) {
			DatasetType type = msg.getDatasetType();
			if(type == DatasetType.STREAM)
				msg.setDatasetType(DatasetType.TIME_SERIES);
		}

		if (log.isInfoEnabled())
			log.info("Registering table="+msg.getModelName());
		if(msg.getDatasetType() != DatasetType.STREAM && msg.getDatasetType() != DatasetType.RELATIONAL_TABLE 
				&& msg.getDatasetType() != DatasetType.TIME_SERIES) {
			if (log.isInfoEnabled())
				log.info("only supporting type of STREAM or RELATIONAL_TABLE or TIME_SERIES right now");
			throw new BadRequest("only supporting type of STREAM or RELATIONAL_TABLE or TIME_SERIES right now");
		} else if(msg.getModelName() == null) {
			if (log.isInfoEnabled())
				log.info("model name was null and is required");
			throw new BadRequest("model name was null and is required");
		} else if(!DboTableMeta.isValidTableName(msg.getModelName())) {
			if (log.isInfoEnabled())
				log.info("modelName is invalid, it much match regular expression=[a-zA-Z_][a-zA-Z_0-9]*");
			throw new BadRequest("modelName is invalid, it much match regular expression=[a-zA-Z_][a-zA-Z_0-9]*");
		}

		String schemaName = null;
		if((msg.getGroups()!=null && msg.getGroups().size()!=0) && msg.getSchema() != null) {
			if (log.isInfoEnabled())
				log.info("use supplied groups and schema attribute...this is not allowed");
			throw new BadRequest("Cannot supply schema AND list of groups.  groups is deprecated, use schema only");
		} else if(msg.getGroups() != null && msg.getGroups().size() > 0) {
			if(msg.getGroups().size() > 1) {
				if (log.isInfoEnabled())
					log.info("more than one group is not allowed");
				throw new BadRequest("Cannot have more than one group anymore.  That is deprecated");
			}

			schemaName = msg.getGroups().get(0);
		} else if(msg.getSchema() != null) {
			schemaName = msg.getSchema();
		} else {
			if (log.isInfoEnabled())
				log.info("missing schema attribute");
			throw new BadRequest("Missing the schema attribute which is required");
		}
		
		DatasetType type = msg.getDatasetType();
		
		validateColumnsForDatasetType(msg);
		ensureTableDoesNotAlreadyExist(msg);
		EntityUser user = SecurityUtil.fetchUser(username, apiKey);

		SecureSchema schema = SecureSchema.findByName(NoSql.em(), schemaName);
		if(schema == null)
			throw new BadRequest("you must specify an existing schema or group");
		
		// Setup the table meta
		DboTableMeta tm = new DboTableMeta();
		if(msg.getDatasetType() != DatasetType.TIME_SERIES) {
			tm.setup(msg.getModelName(), "nreldata", false);
		} else {
			List<DatasetColumnModel> columns = msg.getColumns();
			if(columns.size() != 2)
				throw new BadRequest("TIME_SERIES table can only have two columns, the primary key of long(time since epoch) and value of any type");

			String modelName = msg.getModelName();
			String realCf = Utility.createCfName(modelName);
			long partitionSize = TimeUnit.MILLISECONDS.convert(30, TimeUnit.DAYS);
			tm.setTimeSeries(true);
			tm.setTimeSeriesPartionSize(partitionSize);
			tm.setup(msg.getModelName(), realCf, false);
			tm.setColNameType(long.class);
		}

		// create new Table here and add to security group as well
		SecureTable t = new SecureTable();
		t.setTableName(msg.getModelName());
		t.setDateCreated(new LocalDateTime());
		t.setTableMeta(tm);
		t.setCreator(user);
		t.setIsForLogging(msg.getIsForLogging());
		t.setSearchable(msg.getIsSearchable());
		if (schema != null)
			schema.addTable(t);
		NoSql.em().fillInWithKey(t);
		<commit_msg>add code to insert the type of data into the meta data information<commit_after>package controllers.api;

import gov.nrel.util.SearchUtils;
import gov.nrel.util.Utility;

import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;

import javax.xml.parsers.ParserConfigurationException;

import models.DataTypeEnum;
import models.Entity;
import models.EntityGroup;
import models.EntityGroupXref;
import models.EntityUser;
import models.KeyToTableName;
import models.PermissionType;
import models.RoleMapping;
import models.SdiColumn;
import models.SdiColumnProperty;
import models.SecureResource;
import models.SecureResourceGroupXref;
import models.SecureSchema;
import models.SecureTable;
import models.SecurityGroup;
import models.message.DatasetColumnModel;
import models.message.DatasetColumnSemanticModel;
import models.message.DatasetType;
import models.message.RegisterMessage;
import models.message.RegisterResponseMessage;
import models.message.SearchableParent;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.CoreAdminResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
import org.joda.time.LocalDateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;

import play.Play;
import play.mvc.Util;
import play.mvc.results.BadRequest;

import com.alvazan.orm.api.base.NoSqlEntityManager;
import com.alvazan.orm.api.z8spi.meta.DboColumnCommonMeta;
import com.alvazan.orm.api.z8spi.meta.DboColumnIdMeta;
import com.alvazan.orm.api.z8spi.meta.DboColumnMeta;
import com.alvazan.orm.api.z8spi.meta.DboColumnToOneMeta;
import com.alvazan.orm.api.z8spi.meta.DboTableMeta;
import com.alvazan.orm.api.z8spi.meta.TypedColumn;
import com.alvazan.play.NoSql;

import controllers.SearchPosting;
import controllers.SecurityUtil;

public class ApiRegistrationImpl {

	private static final Logger log = LoggerFactory.getLogger(ApiRegistrationImpl.class);
	public static int divisor = Integer.MAX_VALUE / 10;

	public static RegisterResponseMessage registerImpl(RegisterMessage msg, String username, String apiKey) {
		String mode = (String) Play.configuration.get("upgrade.mode");
		String requestUrl = null;
		if(mode == null) {
		} if(mode.startsWith("http")) {
			requestUrl = mode;
		} else if("NEW".equals(mode)) {
			DatasetType type = msg.getDatasetType();
			if(type == DatasetType.STREAM)
				msg.setDatasetType(DatasetType.TIME_SERIES);
		}

		if (log.isInfoEnabled())
			log.info("Registering table="+msg.getModelName());
		if(msg.getDatasetType() != DatasetType.STREAM && msg.getDatasetType() != DatasetType.RELATIONAL_TABLE 
				&& msg.getDatasetType() != DatasetType.TIME_SERIES) {
			if (log.isInfoEnabled())
				log.info("only supporting type of STREAM or RELATIONAL_TABLE or TIME_SERIES right now");
			throw new BadRequest("only supporting type of STREAM or RELATIONAL_TABLE or TIME_SERIES right now");
		} else if(msg.getModelName() == null) {
			if (log.isInfoEnabled())
				log.info("model name was null and is required");
			throw new BadRequest("model name was null and is required");
		} else if(!DboTableMeta.isValidTableName(msg.getModelName())) {
			if (log.isInfoEnabled())
				log.info("modelName is invalid, it much match regular expression=[a-zA-Z_][a-zA-Z_0-9]*");
			throw new BadRequest("modelName is invalid, it much match regular expression=[a-zA-Z_][a-zA-Z_0-9]*");
		}

		String schemaName = null;
		if((msg.getGroups()!=null && msg.getGroups().size()!=0) && msg.getSchema() != null) {
			if (log.isInfoEnabled())
				log.info("use supplied groups and schema attribute...this is not allowed");
			throw new BadRequest("Cannot supply schema AND list of groups.  groups is deprecated, use schema only");
		} else if(msg.getGroups() != null && msg.getGroups().size() > 0) {
			if(msg.getGroups().size() > 1) {
				if (log.isInfoEnabled())
					log.info("more than one group is not allowed");
				throw new BadRequest("Cannot have more than one group anymore.  That is deprecated");
			}

			schemaName = msg.getGroups().get(0);
		} else if(msg.getSchema() != null) {
			schemaName = msg.getSchema();
		} else {
			if (log.isInfoEnabled())
				log.info("missing schema attribute");
			throw new BadRequest("Missing the schema attribute which is required");
		}
		
		DatasetType type = msg.getDatasetType();
		
		validateColumnsForDatasetType(msg);
		ensureTableDoesNotAlreadyExist(msg);
		EntityUser user = SecurityUtil.fetchUser(username, apiKey);

		SecureSchema schema = SecureSchema.findByName(NoSql.em(), schemaName);
		if(schema == null)
			throw new BadRequest("you must specify an existing schema or group");
		
		// Setup the table meta
		DboTableMeta tm = new DboTableMeta();
		if(msg.getDatasetType() != DatasetType.TIME_SERIES) {
			tm.setup(msg.getModelName(), "nreldata", false);
		} else {
			List<DatasetColumnModel> columns = msg.getColumns();
			if(columns.size() != 2)
				throw new BadRequest("TIME_SERIES table can only have two columns, the primary key of long(time since epoch) and value of any type");

			String modelName = msg.getModelName();
			String realCf = Utility.createCfName(modelName);
			long partitionSize = TimeUnit.MILLISECONDS.convert(30, TimeUnit.DAYS);
			tm.setTimeSeries(true);
			tm.setTimeSeriesPartionSize(partitionSize);
			tm.setup(msg.getModelName(), realCf, false);
			tm.setColNameType(long.class);
		}

		// create new Table here and add to security group as well
		SecureTable t = new SecureTable();
		if(msg.getDatasetType() == DatasetType.RELATIONAL_TABLE) {
			t.setTypeOfData(DataTypeEnum.RELATIONAL);
		} else if(msg.getDatasetType() == DatasetType.TIME_SERIES) {
			t.setTypeOfData(DataTypeEnum.TIME_SERIES);
		}

		t.setTableName(msg.getModelName());
		t.setDateCreated(new LocalDateTime());
		t.setTableMeta(tm);
		t.setCreator(user);
		t.setIsForLogging(msg.getIsForLogging());
		t.setSearchable(msg.getIsSearchable());
		if (schema != null)
			schema.addTable(t);
		NoSql.em().fillInWithKey(t);
		
{'fraction_non_alphanumeric': 0.11981850342228717, 'fraction_numerical': 0.003230023840652157, 'mean_word_length': 5.16888045540797, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2646624', 'n_tokens_mistral': 4465, 'n_tokens_neox': 4160, 'n_words': 897}