<commit_before>/**
 * This file is part of the CernVM file system.
 */

#define __STDC_FORMAT_MACROS

#include "catalog_mgr_rw.h"

#include <inttypes.h>
#include <unistd.h>

#include <cassert>
#include <cstdio>
#include <cstdlib>
#include <string>

#include "catalog_balancer.h"
#include "catalog_rw.h"
#include "logging.h"
#include "manifest.h"
#include "smalloc.h"
#include "statistics.h"
#include "upload.h"
#include "util/posix.h"

using namespace std;  // NOLINT

namespace catalog {

WritableCatalogManager::WritableCatalogManager(
  const shash::Any          &base_hash,
  const std::string         &stratum0,
  const string              &dir_temp,
  upload::Spooler           *spooler,
  download::DownloadManager *download_manager,
  const uint64_t             catalog_entry_warn_threshold,
  perf::Statistics          *statistics,
  bool                       is_balanceable,
  unsigned                   max_weight,
  unsigned                   min_weight)
  : SimpleCatalogManager(base_hash, stratum0, dir_temp, download_manager,
      statistics)
  , spooler_(spooler)
  , catalog_entry_warn_threshold_(catalog_entry_warn_threshold)
  , is_balanceable_(is_balanceable)
  , max_weight_(max_weight)
  , min_weight_(min_weight)
  , balance_weight_(max_weight / 2)
{
  sync_lock_ =
    reinterpret_cast<pthread_mutex_t *>(smalloc(sizeof(pthread_mutex_t)));
  int retval = pthread_mutex_init(sync_lock_, NULL);
  assert(retval == 0);
}


WritableCatalogManager::~WritableCatalogManager() {
  pthread_mutex_destroy(sync_lock_);
  free(sync_lock_);
}


/**
 * This method is virtual in AbstractCatalogManager.  It returns a new catalog
 * structure in the form the different CatalogManagers need it.
 * In this case it returns a stub for a WritableCatalog.
 * @param mountpoint     the mount point of the catalog stub to create
 * @param catalog_hash   the content hash of the catalog to create
 * @param parent_catalog the parent of the catalog stub to create
 * @return a pointer to the catalog stub structure created
 */
Catalog* WritableCatalogManager::CreateCatalog(
  const PathString &mountpoint,
  const shash::Any &catalog_hash,
  Catalog          *parent_catalog)
{
  return new WritableCatalog(mountpoint.ToString(),
                             catalog_hash,
                             parent_catalog);
}


void WritableCatalogManager::ActivateCatalog(Catalog *catalog) {
  catalog->TakeDatabaseFileOwnership();
}


/**
 * This method is invoked if we create a completely new repository.
 * The new root catalog will already contain a root entry.
 * It is uploaded by a Forklift to the upstream storage.
 * @return true on success, false otherwise
 */
manifest::Manifest *WritableCatalogManager::CreateRepository(
  const string      &dir_temp,
  const bool         volatile_content,
  const std::string &voms_authz,
  upload::Spooler   *spooler)
{
  // Create a new root catalog at file_path
  string file_path = dir_temp + "/new_root_catalog";

  shash::Algorithms hash_algorithm = spooler->GetHashAlgorithm();

  // A newly created catalog always needs a root entry
  // we create and configure this here
  DirectoryEntry root_entry;
  root_entry.inode_             = DirectoryEntry::kInvalidInode;
  root_entry.mode_              = 16877;
  root_entry.size_              = 4096;
  root_entry.mtime_             = time(NULL);
  root_entry.uid_               = getuid();
  root_entry.gid_               = getgid();
  root_entry.checksum_          = shash::Any(hash_algorithm);
  root_entry.linkcount_         = 2;
  string root_path = "";

  // Create the database schema and the inital root entry
  {
    UniquePtr<CatalogDatabase> new_clg_db(CatalogDatabase::Create(file_path));
    if (!new_clg_db.IsValid() ||
        !new_clg_db->InsertInitialValues(root_path,
                                         volatile_content,
                                         voms_authz,
                                         root_entry))
    {
      LogCvmfs(kLogCatalog, kLogStderr, "creation of catalog '%s' failed",
               file_path.c_str());
      return NULL;
    }
  }

  // Compress root catalog;
  int64_t catalog_size = GetFileSize(file_path);
  if (catalog_size < 0) {
    unlink(file_path.c_str());
    return NULL;
  }
  string file_path_compressed = file_path + ".compressed";
  shash::Any hash_catalog(hash_algorithm, shash::kSuffixCatalog);
  bool retval = zlib::CompressPath2Path(file_path, file_path_compressed,
                                        &hash_catalog);
  if (!retval) {
    LogCvmfs(kLogCatalog, kLogStderr, "compression of catalog '%s' failed",
             file_path.c_str());
    unlink(file_path.c_str());
    return NULL;
  }
  unlink(file_path.c_str());

  // Create manifest
  const string manifest_path = dir_temp + "/manifest";
  manifest::Manifest *manifest =
    new manifest::Manifest(hash_catalog, catalog_size, "");
  if (!voms_authz.empty()) {
    manifest->set_has_alt_catalog_path(true);
  }

  // Upload catalog
  spooler->Upload(file_path_compressed, "data/" + hash_catalog.MakePath());
  spooler->WaitForUpload();
  unlink(file_path_compressed.c_str());
  if (spooler->GetNumberOfErrors() > 0) {
    LogCvmfs(kLogCatalog, kLogStderr, "failed to commit catalog %s",
             file_path_compressed.c_str());
    delete manifest;
    return NULL;
  }

  return manifest;
}


/**
 * Tries to retrieve the catalog containing the given path
 * This method is just a wrapper around the FindCatalog method of
 * AbstractCatalogManager to provide a direct interface returning
 * WritableCatalog classes.
 * @param path the path to look for
 * @param result the retrieved catalog (as a pointer)
 * @return true if catalog was found, false otherwise
 */
bool WritableCatalogManager::FindCatalog(const string     &path,
                                         WritableCatalog **result,
                                         DirectoryEntry   *dirent) {
  const PathString ps_path(path);

  Catalog *best_fit =
    AbstractCatalogManager<Catalog>::FindCatalog(ps_path);
  assert(best_fit != NULL);
  Catalog *catalog = NULL;
  bool retval = MountSubtree(ps_path, best_fit, &catalog);
  if (!retval)
    return false;

  catalog::DirectoryEntry dummy;
  if (NULL == dirent) {
    dirent = &dummy;
  }
  bool found = LookupPath(ps_path, kLookupSole, dirent);
  if (!found || !catalog->IsWritable())
    return false;
<commit_msg>Doc: update WritableClgMg::FindCatalog() docs<commit_after>/**
 * This file is part of the CernVM file system.
 */

#define __STDC_FORMAT_MACROS

#include "catalog_mgr_rw.h"

#include <inttypes.h>
#include <unistd.h>

#include <cassert>
#include <cstdio>
#include <cstdlib>
#include <string>

#include "catalog_balancer.h"
#include "catalog_rw.h"
#include "logging.h"
#include "manifest.h"
#include "smalloc.h"
#include "statistics.h"
#include "upload.h"
#include "util/posix.h"

using namespace std;  // NOLINT

namespace catalog {

WritableCatalogManager::WritableCatalogManager(
  const shash::Any          &base_hash,
  const std::string         &stratum0,
  const string              &dir_temp,
  upload::Spooler           *spooler,
  download::DownloadManager *download_manager,
  const uint64_t             catalog_entry_warn_threshold,
  perf::Statistics          *statistics,
  bool                       is_balanceable,
  unsigned                   max_weight,
  unsigned                   min_weight)
  : SimpleCatalogManager(base_hash, stratum0, dir_temp, download_manager,
      statistics)
  , spooler_(spooler)
  , catalog_entry_warn_threshold_(catalog_entry_warn_threshold)
  , is_balanceable_(is_balanceable)
  , max_weight_(max_weight)
  , min_weight_(min_weight)
  , balance_weight_(max_weight / 2)
{
  sync_lock_ =
    reinterpret_cast<pthread_mutex_t *>(smalloc(sizeof(pthread_mutex_t)));
  int retval = pthread_mutex_init(sync_lock_, NULL);
  assert(retval == 0);
}


WritableCatalogManager::~WritableCatalogManager() {
  pthread_mutex_destroy(sync_lock_);
  free(sync_lock_);
}


/**
 * This method is virtual in AbstractCatalogManager.  It returns a new catalog
 * structure in the form the different CatalogManagers need it.
 * In this case it returns a stub for a WritableCatalog.
 * @param mountpoint     the mount point of the catalog stub to create
 * @param catalog_hash   the content hash of the catalog to create
 * @param parent_catalog the parent of the catalog stub to create
 * @return a pointer to the catalog stub structure created
 */
Catalog* WritableCatalogManager::CreateCatalog(
  const PathString &mountpoint,
  const shash::Any &catalog_hash,
  Catalog          *parent_catalog)
{
  return new WritableCatalog(mountpoint.ToString(),
                             catalog_hash,
                             parent_catalog);
}


void WritableCatalogManager::ActivateCatalog(Catalog *catalog) {
  catalog->TakeDatabaseFileOwnership();
}


/**
 * This method is invoked if we create a completely new repository.
 * The new root catalog will already contain a root entry.
 * It is uploaded by a Forklift to the upstream storage.
 * @return true on success, false otherwise
 */
manifest::Manifest *WritableCatalogManager::CreateRepository(
  const string      &dir_temp,
  const bool         volatile_content,
  const std::string &voms_authz,
  upload::Spooler   *spooler)
{
  // Create a new root catalog at file_path
  string file_path = dir_temp + "/new_root_catalog";

  shash::Algorithms hash_algorithm = spooler->GetHashAlgorithm();

  // A newly created catalog always needs a root entry
  // we create and configure this here
  DirectoryEntry root_entry;
  root_entry.inode_             = DirectoryEntry::kInvalidInode;
  root_entry.mode_              = 16877;
  root_entry.size_              = 4096;
  root_entry.mtime_             = time(NULL);
  root_entry.uid_               = getuid();
  root_entry.gid_               = getgid();
  root_entry.checksum_          = shash::Any(hash_algorithm);
  root_entry.linkcount_         = 2;
  string root_path = "";

  // Create the database schema and the inital root entry
  {
    UniquePtr<CatalogDatabase> new_clg_db(CatalogDatabase::Create(file_path));
    if (!new_clg_db.IsValid() ||
        !new_clg_db->InsertInitialValues(root_path,
                                         volatile_content,
                                         voms_authz,
                                         root_entry))
    {
      LogCvmfs(kLogCatalog, kLogStderr, "creation of catalog '%s' failed",
               file_path.c_str());
      return NULL;
    }
  }

  // Compress root catalog;
  int64_t catalog_size = GetFileSize(file_path);
  if (catalog_size < 0) {
    unlink(file_path.c_str());
    return NULL;
  }
  string file_path_compressed = file_path + ".compressed";
  shash::Any hash_catalog(hash_algorithm, shash::kSuffixCatalog);
  bool retval = zlib::CompressPath2Path(file_path, file_path_compressed,
                                        &hash_catalog);
  if (!retval) {
    LogCvmfs(kLogCatalog, kLogStderr, "compression of catalog '%s' failed",
             file_path.c_str());
    unlink(file_path.c_str());
    return NULL;
  }
  unlink(file_path.c_str());

  // Create manifest
  const string manifest_path = dir_temp + "/manifest";
  manifest::Manifest *manifest =
    new manifest::Manifest(hash_catalog, catalog_size, "");
  if (!voms_authz.empty()) {
    manifest->set_has_alt_catalog_path(true);
  }

  // Upload catalog
  spooler->Upload(file_path_compressed, "data/" + hash_catalog.MakePath());
  spooler->WaitForUpload();
  unlink(file_path_compressed.c_str());
  if (spooler->GetNumberOfErrors() > 0) {
    LogCvmfs(kLogCatalog, kLogStderr, "failed to commit catalog %s",
             file_path_compressed.c_str());
    delete manifest;
    return NULL;
  }

  return manifest;
}


/**
 * Retrieve the catalog containing the given path.
 * Other than AbstractCatalogManager::FindCatalog() this mounts nested
 * catalogs if necessary and returns  WritableCatalog objects.
 * Furthermore it optionally returns the looked-up DirectoryEntry.
 *
 * @param path    the path to look for
 * @param result  the retrieved catalog (as a pointer)
 * @param dirent  is set to looked up DirectoryEntry for 'path' if non-NULL
 * @return        true if catalog was found
 */
bool WritableCatalogManager::FindCatalog(const string     &path,
                                         WritableCatalog **result,
                                         DirectoryEntry   *dirent) {
  const PathString ps_path(path);

  Catalog *best_fit =
    AbstractCatalogManager<Catalog>::FindCatalog(ps_path);
  assert(best_fit != NULL);
  Catalog *catalog = NULL;
  bool retval = MountSubtree(ps_path, best_fit, &catalog);
  if (!retval)
    return false;

  catalog::DirectoryEntry dummy;
  if (NULL == dirent) {
    dirent = &dummy;
  }
  bool found = LookupPath(ps_path, kLookupSole, dirent);
  if (!found || !catalog->IsWritable())
    return false;

{'fraction_non_alphanumeric': 0.09474091260634183, 'fraction_numerical': 0.0032482598607888632, 'mean_word_length': 3.054876136719975, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4297103', 'n_tokens_mistral': 3897, 'n_tokens_neox': 3871, 'n_words': 983}