<commit_before>/*******************************************************************************
 * Copyright 2012 Apigee Corporation
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ******************************************************************************/
package org.usergrid.persistence.cassandra;

import static me.prettyprint.cassandra.service.FailoverPolicy.ON_FAIL_TRY_ALL_AVAILABLE;
import static me.prettyprint.hector.api.factory.HFactory.createColumn;
import static me.prettyprint.hector.api.factory.HFactory.createMultigetSliceQuery;
import static me.prettyprint.hector.api.factory.HFactory.createMutator;
import static me.prettyprint.hector.api.factory.HFactory.createRangeSlicesQuery;
import static me.prettyprint.hector.api.factory.HFactory.createSliceQuery;
import static me.prettyprint.hector.api.factory.HFactory.createVirtualKeyspace;
import static org.apache.commons.collections.MapUtils.getIntValue;
import static org.apache.commons.collections.MapUtils.getString;
import static org.usergrid.persistence.cassandra.ApplicationCF.ENTITY_ID_SETS;
import static org.usergrid.persistence.cassandra.CassandraPersistenceUtils.batchExecute;
import static org.usergrid.persistence.cassandra.CassandraPersistenceUtils.buildSetIdListMutator;
import static org.usergrid.utils.ConversionUtils.bytebuffer;
import static org.usergrid.utils.ConversionUtils.bytebuffers;
import static org.usergrid.utils.ConversionUtils.string;
import static org.usergrid.utils.ConversionUtils.uuid;
import static org.usergrid.utils.JsonUtils.mapToFormattedJsonString;
import static org.usergrid.utils.MapUtils.asMap;
import static org.usergrid.utils.MapUtils.filter;

import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;

import me.prettyprint.cassandra.connection.HConnectionManager;
import me.prettyprint.cassandra.model.ConfigurableConsistencyLevel;
import me.prettyprint.cassandra.serializers.ByteBufferSerializer;
import me.prettyprint.cassandra.serializers.BytesArraySerializer;
import me.prettyprint.cassandra.serializers.DynamicCompositeSerializer;
import me.prettyprint.cassandra.serializers.LongSerializer;
import me.prettyprint.cassandra.serializers.StringSerializer;
import me.prettyprint.cassandra.serializers.UUIDSerializer;
import me.prettyprint.cassandra.service.CassandraHostConfigurator;
import me.prettyprint.cassandra.service.ThriftKsDef;
import me.prettyprint.hector.api.Cluster;
import me.prettyprint.hector.api.ConsistencyLevelPolicy;
import me.prettyprint.hector.api.HConsistencyLevel;
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.Serializer;
import me.prettyprint.hector.api.beans.ColumnSlice;
import me.prettyprint.hector.api.beans.DynamicComposite;
import me.prettyprint.hector.api.beans.HColumn;
import me.prettyprint.hector.api.beans.OrderedRows;
import me.prettyprint.hector.api.beans.Row;
import me.prettyprint.hector.api.beans.Rows;
import me.prettyprint.hector.api.ddl.ColumnDefinition;
import me.prettyprint.hector.api.ddl.ColumnFamilyDefinition;
import me.prettyprint.hector.api.ddl.KeyspaceDefinition;
import me.prettyprint.hector.api.factory.HFactory;
import me.prettyprint.hector.api.mutation.Mutator;
import me.prettyprint.hector.api.query.ColumnQuery;
import me.prettyprint.hector.api.query.CountQuery;
import me.prettyprint.hector.api.query.MultigetSliceQuery;
import me.prettyprint.hector.api.query.QueryResult;
import me.prettyprint.hector.api.query.RangeSlicesQuery;
import me.prettyprint.hector.api.query.SliceQuery;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.usergrid.locking.LockManager;
import org.usergrid.persistence.IndexBucketLocator;
import org.usergrid.persistence.IndexBucketLocator.IndexType;
import org.usergrid.utils.JsonUtils;

public class CassandraService {

    public static String SYSTEM_KEYSPACE = "Usergrid";

    public static String STATIC_APPLICATION_KEYSPACE = "Usergrid_Applications";

    public static final boolean USE_VIRTUAL_KEYSPACES = true;

    public static final String APPLICATIONS_CF = "Applications";
    public static final String PROPERTIES_CF = "Properties";
    public static final String TOKENS_CF = "Tokens";
    public static final String PRINCIPAL_TOKEN_CF = "PrincipalTokens";

    public static final int DEFAULT_COUNT = 1000;
    public static final int ALL_COUNT = 100000;
    public static final int INDEX_ENTRY_LIST_COUNT = 1000;
    public static final int DEFAULT_SEARCH_COUNT = 10000;

    public static final int RETRY_COUNT = 5;

    public static final String DEFAULT_APPLICATION = "default-app";
    public static final String DEFAULT_ORGANIZATION = "usergrid";
    public static final String MANAGEMENT_APPLICATION = "management";

    public static final UUID MANAGEMENT_APPLICATION_ID = new UUID(0, 1);
    public static final UUID DEFAULT_APPLICATION_ID = new UUID(0, 16);

    private static final Logger logger = LoggerFactory
            .getLogger(CassandraService.class);

    private static final Logger db_logger = LoggerFactory
            .getLogger(CassandraService.class.getPackage().getName() + ".DB");

    Cluster cluster;
    CassandraHostConfigurator chc;
    Properties properties;
    LockManager lockManager;

    ConsistencyLevelPolicy consistencyLevelPolicy;

    private Keyspace systemKeyspace;

    private Map<String, String> accessMap;

    public static final StringSerializer se = new StringSerializer();
    public static final ByteBufferSerializer be = new ByteBufferSerializer();
    public static final UUIDSerializer ue = new UUIDSerializer();
    public static final BytesArraySerializer bae = new BytesArraySerializer();
    public static final DynamicCompositeSerializer dce = new DynamicCompositeSerializer();
    public static final LongSerializer le = new LongSerializer();

    public static final UUID NULL_ID = new UUID(0, 0);

    public CassandraService(Properties properties, Cluster cluster,
            CassandraHostConfigurator cassandraHostConfigurator,
            LockManager lockManager) {
        this.properties = properties;
        this.cluster = cluster;
        chc = cassandraHostConfigurator;
        this.lockManager = lockManager;
        db_logger.info("" + cluster.getKnownPoolHosts(false));
    }

    public void init() throws Exception {
        HFactory.getOrCreateCluster(cluster.getName(), chc);
        if (consistencyLevelPolicy == null) {
            consistencyLevelPolicy = new ConfigurableConsistencyLevel();
            ((ConfigurableConsistencyLevel) consistencyLevelPolicy)
                    .setDefaultReadConsistencyLevel(HConsistencyLevel.ONE);
        }
        accessMap = new HashMap<String, String>(2);
        accessMap.put("username", properties.getProperty("cassandra.username"));
        accessMap.put("password", properties.getProperty("<PASSWORD>.password"));
        systemKeyspace = HFactory.createKeyspace(SYSTEM_KEYSPACE, cluster,
                consistencyLevelPolicy, ON_FAIL_TRY_ALL_AVAILABLE, accessMap);
    }

    public Cluster getCluster() {
        return cluster;
    }

    public void setCluster(Cluster cluster) {
        this.cluster = cluster;
    }

    public CassandraHostConfigurator getCassandraHostConfigurator() {
        return chc;
    }

    public void setCassandraHostConfigurator(CassandraHostConfigurator chc) {
        this.chc = chc;
    }<commit_msg>removed redundant getOrCreateClsuter call<commit_after>/*******************************************************************************
 * Copyright 2012 Apigee Corporation
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ******************************************************************************/
package org.usergrid.persistence.cassandra;

import static me.prettyprint.cassandra.service.FailoverPolicy.ON_FAIL_TRY_ALL_AVAILABLE;
import static me.prettyprint.hector.api.factory.HFactory.createColumn;
import static me.prettyprint.hector.api.factory.HFactory.createMultigetSliceQuery;
import static me.prettyprint.hector.api.factory.HFactory.createMutator;
import static me.prettyprint.hector.api.factory.HFactory.createRangeSlicesQuery;
import static me.prettyprint.hector.api.factory.HFactory.createSliceQuery;
import static me.prettyprint.hector.api.factory.HFactory.createVirtualKeyspace;
import static org.apache.commons.collections.MapUtils.getIntValue;
import static org.apache.commons.collections.MapUtils.getString;
import static org.usergrid.persistence.cassandra.ApplicationCF.ENTITY_ID_SETS;
import static org.usergrid.persistence.cassandra.CassandraPersistenceUtils.batchExecute;
import static org.usergrid.persistence.cassandra.CassandraPersistenceUtils.buildSetIdListMutator;
import static org.usergrid.utils.ConversionUtils.bytebuffer;
import static org.usergrid.utils.ConversionUtils.bytebuffers;
import static org.usergrid.utils.ConversionUtils.string;
import static org.usergrid.utils.ConversionUtils.uuid;
import static org.usergrid.utils.JsonUtils.mapToFormattedJsonString;
import static org.usergrid.utils.MapUtils.asMap;
import static org.usergrid.utils.MapUtils.filter;

import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;

import me.prettyprint.cassandra.connection.HConnectionManager;
import me.prettyprint.cassandra.model.ConfigurableConsistencyLevel;
import me.prettyprint.cassandra.serializers.ByteBufferSerializer;
import me.prettyprint.cassandra.serializers.BytesArraySerializer;
import me.prettyprint.cassandra.serializers.DynamicCompositeSerializer;
import me.prettyprint.cassandra.serializers.LongSerializer;
import me.prettyprint.cassandra.serializers.StringSerializer;
import me.prettyprint.cassandra.serializers.UUIDSerializer;
import me.prettyprint.cassandra.service.CassandraHostConfigurator;
import me.prettyprint.cassandra.service.ThriftKsDef;
import me.prettyprint.hector.api.Cluster;
import me.prettyprint.hector.api.ConsistencyLevelPolicy;
import me.prettyprint.hector.api.HConsistencyLevel;
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.Serializer;
import me.prettyprint.hector.api.beans.ColumnSlice;
import me.prettyprint.hector.api.beans.DynamicComposite;
import me.prettyprint.hector.api.beans.HColumn;
import me.prettyprint.hector.api.beans.OrderedRows;
import me.prettyprint.hector.api.beans.Row;
import me.prettyprint.hector.api.beans.Rows;
import me.prettyprint.hector.api.ddl.ColumnDefinition;
import me.prettyprint.hector.api.ddl.ColumnFamilyDefinition;
import me.prettyprint.hector.api.ddl.KeyspaceDefinition;
import me.prettyprint.hector.api.factory.HFactory;
import me.prettyprint.hector.api.mutation.Mutator;
import me.prettyprint.hector.api.query.ColumnQuery;
import me.prettyprint.hector.api.query.CountQuery;
import me.prettyprint.hector.api.query.MultigetSliceQuery;
import me.prettyprint.hector.api.query.QueryResult;
import me.prettyprint.hector.api.query.RangeSlicesQuery;
import me.prettyprint.hector.api.query.SliceQuery;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.usergrid.locking.LockManager;
import org.usergrid.persistence.IndexBucketLocator;
import org.usergrid.persistence.IndexBucketLocator.IndexType;
import org.usergrid.utils.JsonUtils;

public class CassandraService {

    public static String SYSTEM_KEYSPACE = "Usergrid";

    public static String STATIC_APPLICATION_KEYSPACE = "Usergrid_Applications";

    public static final boolean USE_VIRTUAL_KEYSPACES = true;

    public static final String APPLICATIONS_CF = "Applications";
    public static final String PROPERTIES_CF = "Properties";
    public static final String TOKENS_CF = "Tokens";
    public static final String PRINCIPAL_TOKEN_CF = "PrincipalTokens";

    public static final int DEFAULT_COUNT = 1000;
    public static final int ALL_COUNT = 100000;
    public static final int INDEX_ENTRY_LIST_COUNT = 1000;
    public static final int DEFAULT_SEARCH_COUNT = 10000;

    public static final int RETRY_COUNT = 5;

    public static final String DEFAULT_APPLICATION = "default-app";
    public static final String DEFAULT_ORGANIZATION = "usergrid";
    public static final String MANAGEMENT_APPLICATION = "management";

    public static final UUID MANAGEMENT_APPLICATION_ID = new UUID(0, 1);
    public static final UUID DEFAULT_APPLICATION_ID = new UUID(0, 16);

    private static final Logger logger = LoggerFactory
            .getLogger(CassandraService.class);

    private static final Logger db_logger = LoggerFactory
            .getLogger(CassandraService.class.getPackage().getName() + ".DB");

    Cluster cluster;
    CassandraHostConfigurator chc;
    Properties properties;
    LockManager lockManager;

    ConsistencyLevelPolicy consistencyLevelPolicy;

    private Keyspace systemKeyspace;

    private Map<String, String> accessMap;

    public static final StringSerializer se = new StringSerializer();
    public static final ByteBufferSerializer be = new ByteBufferSerializer();
    public static final UUIDSerializer ue = new UUIDSerializer();
    public static final BytesArraySerializer bae = new BytesArraySerializer();
    public static final DynamicCompositeSerializer dce = new DynamicCompositeSerializer();
    public static final LongSerializer le = new LongSerializer();

    public static final UUID NULL_ID = new UUID(0, 0);

    public CassandraService(Properties properties, Cluster cluster,
            CassandraHostConfigurator cassandraHostConfigurator,
            LockManager lockManager) {
        this.properties = properties;
        this.cluster = cluster;
        chc = cassandraHostConfigurator;
        this.lockManager = lockManager;
        db_logger.info("" + cluster.getKnownPoolHosts(false));
    }

    public void init() throws Exception {
        if (consistencyLevelPolicy == null) {
            consistencyLevelPolicy = new ConfigurableConsistencyLevel();
            ((ConfigurableConsistencyLevel) consistencyLevelPolicy)
                    .setDefaultReadConsistencyLevel(HConsistencyLevel.ONE);
        }
        accessMap = new HashMap<String, String>(2);
        accessMap.put("username", properties.getProperty("cassandra.username"));
        accessMap.put("password", <PASSWORD>("<PASSWORD>"));
        systemKeyspace = HFactory.createKeyspace(SYSTEM_KEYSPACE, cluster,
                consistencyLevelPolicy, ON_FAIL_TRY_ALL_AVAILABLE, accessMap);
    }

    public Cluster getCluster() {
        return cluster;
    }

    public void setCluster(Cluster cluster) {
        this.cluster = cluster;
    }

    public CassandraHostConfigurator getCassandraHostConfigurator() {
        return chc;
    }

    public void setCassandraHostConfigurator(CassandraHostConfigurator chc) {
        this.chc = chc;
    }
{'fraction_non_alphanumeric': 0.0982120989468528, 'fraction_numerical': 0.004653441097232427, 'mean_word_length': 5.026937269372694, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2374179', 'n_tokens_mistral': 4710, 'n_tokens_neox': 4478, 'n_words': 1009}