<commit_before>/**
 * This file is part of the CernVM File System.
 */

#define __STDC_FORMAT_MACROS

#include "swissknife_scrub.h"
#include "cvmfs_config.h"

#include "fs_traversal.h"
#include "logging.h"
#include "smalloc.h"
#include "util/posix.h"
#include "util/string.h"

using namespace std;  // NOLINT

namespace swissknife {

const size_t kHashSubtreeLength = 2;
const std::string kTxnDirectoryName = "txn";

CommandScrub::CommandScrub()
  : machine_readable_output_(false)
  , alerts_(0)
{
  int retval = pthread_mutex_init(&alerts_mutex_, NULL);
  assert(retval == 0);
}


CommandScrub::~CommandScrub() {
  pthread_mutex_destroy(&alerts_mutex_);
}


swissknife::ParameterList CommandScrub::GetParams() const {
  swissknife::ParameterList r;
  r.push_back(Parameter::Mandatory('r', "repository directory"));
  r.push_back(Parameter::Switch('m', "machine readable output"));
  return r;
}

const char *CommandScrub::Alerts::ToString(const CommandScrub::Alerts::Type t) {
  switch (t) {
    case Alerts::kUnexpectedFile:
      return "unexpected regular file";
    case Alerts::kUnexpectedSymlink:
      return "unexpected symlink";
    case Alerts::kUnexpectedSubdir:
      return "unexpected subdir in CAS subdir";
    case Alerts::kUnexpectedModifier:
      return "unknown object modifier";
    case Alerts::kMalformedHash:
      return "malformed content hash";
    case Alerts::kMalformedCasSubdir:
      return "malformed CAS subdir length";
    case Alerts::kContentHashMismatch:
      return "mismatch of file name and content hash";
    default:
      return "unknown alert";
  }
}

void CommandScrub::FileCallback(
  const std::string &relative_path,
  const std::string &file_name)
{
  assert(!file_name.empty());

  if (relative_path.empty()) {
    PrintAlert(Alerts::kUnexpectedFile, repo_path_ + "/" + file_name);
    return;
  }
  if (relative_path == kTxnDirectoryName) {
    // transaction directory should be ignored
    return;
  }

  const string full_path = MakeFullPath(relative_path, file_name);
  const std::string hash_string =
      CheckPathAndExtractHash(relative_path, file_name, full_path);
  if (hash_string.empty()) {
    return;
  }

  if (!shash::HexPtr(hash_string).IsValid()) {
    PrintAlert(Alerts::kMalformedHash, full_path, hash_string);
    return;
  }

  shash::Any hash_from_name =
    shash::MkFromSuffixedHexPtr(shash::HexPtr(hash_string));
  IngestionSource* full_path_source = new FileIngestionSource(full_path);
  pipeline_scrubbing_.Process(
    full_path_source,
    hash_from_name.algorithm,
    hash_from_name.suffix);
}


void CommandScrub::DirCallback(
  const std::string &relative_path,
  const std::string &dir_name)
{
  const string full_path = MakeFullPath(relative_path, dir_name);
  // Check for nested subdirs
  if (relative_path.size() > 0) {
    PrintAlert(Alerts::kUnexpectedSubdir, full_path);
    return;
  }

  // Check CAS hash subdirectory name length
  if (!dir_name.empty() && dir_name.size() != kHashSubtreeLength &&
      dir_name != kTxnDirectoryName) {
    PrintAlert(Alerts::kMalformedCasSubdir, full_path);
  }
}

void CommandScrub::SymlinkCallback(const std::string &relative_path,
                                   const std::string &symlink_name) {
  const string full_path = MakeFullPath(relative_path, symlink_name);
  PrintAlert(Alerts::kUnexpectedSymlink, full_path);
}
<commit_msg>Whitelist data/txn/receiver in swissknife scrub<commit_after>/**
 * This file is part of the CernVM File System.
 */

#define __STDC_FORMAT_MACROS

#include "swissknife_scrub.h"
#include "cvmfs_config.h"

#include "fs_traversal.h"
#include "logging.h"
#include "smalloc.h"
#include "util/posix.h"
#include "util/string.h"

using namespace std;  // NOLINT

namespace swissknife {

const size_t kHashSubtreeLength = 2;
const std::string kTxnDirectoryName = "txn";

CommandScrub::CommandScrub()
  : machine_readable_output_(false)
  , alerts_(0)
{
  int retval = pthread_mutex_init(&alerts_mutex_, NULL);
  assert(retval == 0);
}


CommandScrub::~CommandScrub() {
  pthread_mutex_destroy(&alerts_mutex_);
}


swissknife::ParameterList CommandScrub::GetParams() const {
  swissknife::ParameterList r;
  r.push_back(Parameter::Mandatory('r', "repository directory"));
  r.push_back(Parameter::Switch('m', "machine readable output"));
  return r;
}

const char *CommandScrub::Alerts::ToString(const CommandScrub::Alerts::Type t) {
  switch (t) {
    case Alerts::kUnexpectedFile:
      return "unexpected regular file";
    case Alerts::kUnexpectedSymlink:
      return "unexpected symlink";
    case Alerts::kUnexpectedSubdir:
      return "unexpected subdir in CAS subdir";
    case Alerts::kUnexpectedModifier:
      return "unknown object modifier";
    case Alerts::kMalformedHash:
      return "malformed content hash";
    case Alerts::kMalformedCasSubdir:
      return "malformed CAS subdir length";
    case Alerts::kContentHashMismatch:
      return "mismatch of file name and content hash";
    default:
      return "unknown alert";
  }
}

void CommandScrub::FileCallback(
  const std::string &relative_path,
  const std::string &file_name)
{
  assert(!file_name.empty());

  if (relative_path.empty()) {
    PrintAlert(Alerts::kUnexpectedFile, repo_path_ + "/" + file_name);
    return;
  }
  if (relative_path == kTxnDirectoryName) {
    // transaction directory should be ignored
    return;
  }

  const string full_path = MakeFullPath(relative_path, file_name);
  const std::string hash_string =
      CheckPathAndExtractHash(relative_path, file_name, full_path);
  if (hash_string.empty()) {
    return;
  }

  if (!shash::HexPtr(hash_string).IsValid()) {
    PrintAlert(Alerts::kMalformedHash, full_path, hash_string);
    return;
  }

  shash::Any hash_from_name =
    shash::MkFromSuffixedHexPtr(shash::HexPtr(hash_string));
  IngestionSource* full_path_source = new FileIngestionSource(full_path);
  pipeline_scrubbing_.Process(
    full_path_source,
    hash_from_name.algorithm,
    hash_from_name.suffix);
}


void CommandScrub::DirCallback(
  const std::string &relative_path,
  const std::string &dir_name)
{
  const string full_path = MakeFullPath(relative_path, dir_name);

  // The directory "/srv/cvmfs/<REPO_NAME>/data/txn/receiver" is whitelisted
  if (HasSuffix(full_path, "data/txn/receiver", false)) {
    return;
  }

  // Check for nested subdirs
  if (relative_path.size() > 0) {
    PrintAlert(Alerts::kUnexpectedSubdir, full_path);
    return;
  }

  // Check CAS hash subdirectory name length
  if (!dir_name.empty() && dir_name.size() != kHashSubtreeLength &&
      dir_name != kTxnDirectoryName) {
    PrintAlert(Alerts::kMalformedCasSubdir, full_path);
  }
}

void CommandScrub::SymlinkCallback(const std::string &relative_path,
                                   const std::string &symlink_name) {
  const string full_path = MakeFullPath(relative_path, symlink_name);
  PrintAlert(Alerts::kUnexpectedSymlink, full_path);
}

{'fraction_non_alphanumeric': 0.11572272661766825, 'fraction_numerical': 0.0011529038766392853, 'mean_word_length': 4.569823434991974, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7556254', 'n_tokens_mistral': 2330, 'n_tokens_neox': 2294, 'n_words': 474}