<commit_before>// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.cloudera.impala.catalog;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.log4j.Logger;

import com.cloudera.impala.analysis.FunctionName;
import com.cloudera.impala.catalog.MetaStoreClientPool.MetaStoreClient;
import com.cloudera.impala.common.ImpalaException;
import com.cloudera.impala.thrift.TCatalogObject;
import com.cloudera.impala.thrift.TFunctionType;
import com.cloudera.impala.thrift.TPartitionKeyValue;
import com.cloudera.impala.thrift.TTableName;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.cache.CacheLoader;
import com.google.common.collect.Lists;

/**
 * Thread safe interface for reading and updating metadata stored in the Hive MetaStore.
 * This class caches db-, table- and column-related metadata. Each time one of these
 * catalog objects is updated/added/removed, the catalogVersion_ is incremented.
 * Although this class is thread safe, it does not guarantee consistency with the
 * MetaStore. It is important to keep in mind that there may be external (potentially
 * conflicting) concurrent metastore updates occurring at any time.
 * All reads and writes of catalog objects are synchronized using the catalogLock_. To
 * perform atomic bulk operations on the Catalog, the getReadLock()/getWriteLock()
 * functions can be leveraged.
 */
public abstract class Catalog {
  // Initial catalog version.
  public final static long INITIAL_CATALOG_VERSION = 0L;
  public static final String DEFAULT_DB = "default";

  private static final Logger LOG = Logger.getLogger(Catalog.class);

  // Last assigned catalog version. Starts at INITIAL_CATALOG_VERSION and incremented
  // with each update to the Catalog. Persisted across the lifetime of the Catalog.
  // Atomic to ensure versions are always sequentially increasing, even when updated
  // from different threads.
  // TODO: This probably doesn't need to be atomic and can be updated while holding
  // the catalogLock_.
  private final static AtomicLong catalogVersion_ =
      new AtomicLong(INITIAL_CATALOG_VERSION);
  private static final int META_STORE_CLIENT_POOL_SIZE = 5;
  private final MetaStoreClientPool metaStoreClientPool_ = new MetaStoreClientPool(0);
  private final CatalogInitStrategy initStrategy_;
  private final AtomicInteger nextTableId_ = new AtomicInteger(0);

  // Cache of database metadata.
  protected final CatalogObjectCache<Db> dbCache_ = new CatalogObjectCache<Db>(
      new CacheLoader<String, Db>() {
        @Override
        public Db load(String dbName) {
          MetaStoreClient msClient = getMetaStoreClient();
          try {
            return Db.loadDb(Catalog.this, msClient.getHiveClient(),
                dbName.toLowerCase(), true);
          } finally {
            msClient.release();
          }
        }
      });

  // Fair lock used to synchronize catalog accesses and updates.
  protected final ReentrantReadWriteLock catalogLock_ =
      new ReentrantReadWriteLock(true);

  // Determines how the Catalog should be initialized.
  public enum CatalogInitStrategy {
    // Load only db and table names on startup.
    LAZY,
    // Load all metadata on startup
    IMMEDIATE,
    // Don't load anything on startup (creates an empty catalog).
    EMPTY,
  }

  /**
   * Creates a new instance of the Catalog, initializing it based on
   * the given CatalogInitStrategy.
   */
  public Catalog(CatalogInitStrategy initStrategy) {
    this.initStrategy_ = initStrategy;
    this.metaStoreClientPool_.addClients(META_STORE_CLIENT_POOL_SIZE);
    reset();
  }

  public Catalog() { this(CatalogInitStrategy.LAZY); }

  /**
   * Adds a database name to the metadata cache and marks the metadata as
   * uninitialized. Used by CREATE DATABASE statements.<commit_msg>Don't add any connections to the Metastore Connection Pool for impalads<commit_after>// Copyright 2012 Cloudera Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.cloudera.impala.catalog;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.log4j.Logger;

import com.cloudera.impala.analysis.FunctionName;
import com.cloudera.impala.catalog.MetaStoreClientPool.MetaStoreClient;
import com.cloudera.impala.common.ImpalaException;
import com.cloudera.impala.thrift.TCatalogObject;
import com.cloudera.impala.thrift.TFunctionType;
import com.cloudera.impala.thrift.TPartitionKeyValue;
import com.cloudera.impala.thrift.TTableName;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.cache.CacheLoader;
import com.google.common.collect.Lists;

/**
 * Thread safe interface for reading and updating metadata stored in the Hive MetaStore.
 * This class caches db-, table- and column-related metadata. Each time one of these
 * catalog objects is updated/added/removed, the catalogVersion_ is incremented.
 * Although this class is thread safe, it does not guarantee consistency with the
 * MetaStore. It is important to keep in mind that there may be external (potentially
 * conflicting) concurrent metastore updates occurring at any time.
 * All reads and writes of catalog objects are synchronized using the catalogLock_. To
 * perform atomic bulk operations on the Catalog, the getReadLock()/getWriteLock()
 * functions can be leveraged.
 */
public abstract class Catalog {
  // Initial catalog version.
  public final static long INITIAL_CATALOG_VERSION = 0L;
  public static final String DEFAULT_DB = "default";

  private static final Logger LOG = Logger.getLogger(Catalog.class);

  // Last assigned catalog version. Starts at INITIAL_CATALOG_VERSION and incremented
  // with each update to the Catalog. Persisted across the lifetime of the Catalog.
  // Atomic to ensure versions are always sequentially increasing, even when updated
  // from different threads.
  // TODO: This probably doesn't need to be atomic and can be updated while holding
  // the catalogLock_.
  private final static AtomicLong catalogVersion_ =
      new AtomicLong(INITIAL_CATALOG_VERSION);
  private static final int META_STORE_CLIENT_POOL_SIZE = 5;
  private final MetaStoreClientPool metaStoreClientPool_ = new MetaStoreClientPool(0);
  private final CatalogInitStrategy initStrategy_;
  private final AtomicInteger nextTableId_ = new AtomicInteger(0);

  // Cache of database metadata.
  protected final CatalogObjectCache<Db> dbCache_ = new CatalogObjectCache<Db>(
      new CacheLoader<String, Db>() {
        @Override
        public Db load(String dbName) {
          MetaStoreClient msClient = getMetaStoreClient();
          try {
            return Db.loadDb(Catalog.this, msClient.getHiveClient(),
                dbName.toLowerCase(), true);
          } finally {
            msClient.release();
          }
        }
      });

  // Fair lock used to synchronize catalog accesses and updates.
  protected final ReentrantReadWriteLock catalogLock_ =
      new ReentrantReadWriteLock(true);

  // Determines how the Catalog should be initialized.
  public enum CatalogInitStrategy {
    // Load only db and table names on startup.
    LAZY,
    // Load all metadata on startup
    IMMEDIATE,
    // Don't load anything on startup (creates an empty catalog).
    EMPTY,
  }

  /**
   * Creates a new instance of the Catalog, initializing it based on
   * the given CatalogInitStrategy.
   */
  public Catalog(CatalogInitStrategy initStrategy) {
    initStrategy_ = initStrategy;
    // Don't create any metastore clients for empty catalogs.
    if (initStrategy != CatalogInitStrategy.EMPTY) {
      metaStoreClientPool_.addClients(META_STORE_CLIENT_POOL_SIZE);
    }
    reset();
  }

  public Catalog() { this(CatalogInitStrategy.LAZY); }

  /**
   * Adds a database name to the metadata cache and marks the metadata as
   * uninitialized. Used by CREATE DATABASE statements.
{'fraction_non_alphanumeric': 0.0761668218979613, 'fraction_numerical': 0.002690675773569285, 'mean_word_length': 3.8808080808080807, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '253620', 'n_tokens_mistral': 2677, 'n_tokens_neox': 2599, 'n_words': 928}